diff --git a/.github/workflows/auto-add-issues.yml b/.github/workflows/auto-add-issues.yml deleted file mode 100644 index a3545a48e..000000000 --- a/.github/workflows/auto-add-issues.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Add issues to beta project - -on: - issues: - types: - - opened - - transferred - -jobs: - add-to-project: - name: Add issue to project - runs-on: ubuntu-22.04 - steps: - - uses: actions/add-to-project@v1.0.2 - with: - project-url: https://github.com/orgs/dandi/projects/16 - github-token: ${{ secrets.AUTO_ADD_ISSUES }} diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index 73684de28..737cb6cd0 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -8,15 +8,15 @@ on: - master paths-ignore: - "web/**" - schedule: - - cron: "0 0 * * *" +# schedule: +# - cron: "0 0 * * *" jobs: test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - tox-env: [lint, type, test, check-migrations] + tox-env: [test, check-migrations] # [lint, type, test, check-migrations] services: postgres: image: postgres:latest diff --git a/.github/workflows/backend-production-deploy.yml b/.github/workflows/backend-production-deploy.yml index f32db2220..7a71ec79e 100644 --- a/.github/workflows/backend-production-deploy.yml +++ b/.github/workflows/backend-production-deploy.yml @@ -1,8 +1,13 @@ name: Deploy backend to production +# This is a temporary workflow used by the LINC project that differs from the "release" strategy in the DANDI Archive ecosystem +# +# The reason for the difference is that the LINC project is still a work-in-progress, thus we want updates to be not tied to version updates + on: - release: - types: [released] + workflow_dispatch: + branches: + - release concurrency: # If this workflow is already running, cancel it to avoid a scenario @@ -12,35 +17,20 @@ concurrency: cancel-in-progress: true jobs: - reset-release-branch: - name: Update release branch - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # fetch history for all branches and tags - token: ${{ secrets.GH_TOKEN }} # use PAT with permissions to push to master - ref: release - - - name: Perform reset - run: | - git reset --hard ${{ github.ref }} - git push --force origin release - - production-deploy: + deploy: name: Deploy to Heroku - runs-on: ubuntu-22.04 - needs: reset-release-branch + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 # fetch history for all branches and tags - ref: release - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" + - name: Check allowed users for deploying to production + run: | + if [[ ! "aaronkanzer kabilar" =~ "${{ github.actor }}" ]]; then + echo "Error: User ${{ github.actor }} is not allowed to deploy." + exit 1 + fi - name: Install Heroku CLI run: curl https://cli-assets.heroku.com/install.sh | sh @@ -54,7 +44,7 @@ jobs: python -m build --sdist - name: Create Heroku Build - run: heroku builds:create -a dandi-api --source-tar dist/*.tar.gz + run: heroku builds:create -a ${{ secrets.HEROKU_PRODUCTION_APP_NAME }} --source-tar dist/*.tar.gz env: HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} HEROKU_EMAIL: ${{ secrets.HEROKU_EMAIL }} diff --git a/.github/workflows/backend-staging-deploy.yml b/.github/workflows/backend-staging-deploy.yml index 58e71d3b3..ad6eb8d93 100644 --- a/.github/workflows/backend-staging-deploy.yml +++ b/.github/workflows/backend-staging-deploy.yml @@ -3,7 +3,7 @@ name: Deploy backend to staging on: push: branches: - - master + - staging paths-ignore: - "web/**" - "CHANGELOG.md" @@ -18,17 +18,12 @@ concurrency: jobs: deploy: name: Deploy to Heroku - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 # fetch history for all branches and tags - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install Heroku CLI run: curl https://cli-assets.heroku.com/install.sh | sh @@ -41,7 +36,7 @@ jobs: python -m build --sdist - name: Create Heroku Build - run: heroku builds:create -a dandi-api-staging --source-tar dist/*.tar.gz + run: heroku builds:create -a linc-brain-staging --source-tar dist/*.tar.gz env: HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} HEROKU_EMAIL: ${{ secrets.HEROKU_EMAIL }} diff --git a/.github/workflows/cli-integration.yml b/.github/workflows/cli-integration.yml index f896e07c4..72e093d7f 100644 --- a/.github/workflows/cli-integration.yml +++ b/.github/workflows/cli-integration.yml @@ -1,4 +1,4 @@ -name: Test Integration with dandi-cli +name: Test Integration with lincbrain-cli on: push: @@ -12,25 +12,21 @@ on: jobs: build-image: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Check out this repository uses: actions/checkout@v4 - name: Build Docker image run: | - docker login -u "$DOCKER_LOGIN" --password-stdin <<<"$DOCKER_TOKEN" docker build \ - -t dandiarchive/dandiarchive-api \ + -t lincbrain/dandiarchive-api \ -f dev/django-public.Dockerfile \ . - env: - DOCKER_LOGIN: ${{ secrets.DOCKER_LOGIN }} - DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} - name: Export Docker image run: | - docker image save -o dandiarchive-api.tgz dandiarchive/dandiarchive-api + docker image save -o dandiarchive-api.tgz lincbrain/dandiarchive-api - name: Upload Docker image tarball uses: actions/upload-artifact@v4 @@ -39,7 +35,7 @@ jobs: path: dandiarchive-api.tgz test: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest needs: build-image strategy: fail-fast: false @@ -72,28 +68,32 @@ jobs: - name: Install released dandi if: matrix.dandi-version == 'release' - run: pip install "dandi[test]" + run: pip install "lincbrain-cli[test]" + + - name: Install dev dandi + if: matrix.dandi-version == 'master' + run: pip install "lincbrain-cli[test] @ git+https://github.com/lincbrain/linc-cli" - name: Install dev dandi if: matrix.dandi-version == 'master' - run: pip install "dandi[test] @ git+https://github.com/dandi/dandi-cli" + run: pip install "lincbrain-cli[test] @ git+https://github.com/lincbrain/linc-cli" - - name: Run dandi-api tests in dandi-cli + - name: Run dandi-api tests in linc-cli run: | python -m pytest --dandi-api \ - "$pythonLocation/lib/python${{ matrix.python }}/site-packages/dandi" + "$pythonLocation/lib/python${{ matrix.python }}/site-packages/lincbrain" env: DANDI_TESTS_PERSIST_DOCKER_COMPOSE: "1" - name: Dump Docker Compose logs if: failure() run: | - docker compose \ - -f "$pythonLocation/lib/python${{ matrix.python }}/site-packages/dandi/tests/data/dandiarchive-docker/docker-compose.yml" \ + docker-compose \ + -f "$pythonLocation/lib/python${{ matrix.python }}/site-packages/lincbrain/tests/data/dandiarchive-docker/docker-compose.yml" \ logs --timestamps - name: Shut down Docker Compose run: | - docker compose \ - -f "$pythonLocation/lib/python${{ matrix.python }}/site-packages/dandi/tests/data/dandiarchive-docker/docker-compose.yml" \ + docker-compose \ + -f "$pythonLocation/lib/python${{ matrix.python }}/site-packages/lincbrain/tests/data/dandiarchive-docker/docker-compose.yml" \ down -v diff --git a/.github/workflows/frontend-ci.yml b/.github/workflows/frontend-ci.yml index 59bccb77c..aaca6d576 100644 --- a/.github/workflows/frontend-ci.yml +++ b/.github/workflows/frontend-ci.yml @@ -4,14 +4,14 @@ on: push: branches: - master - schedule: - - cron: "0 0 * * *" +# schedule: +# - cron: "0 0 * * *" jobs: lint-type-check: defaults: run: working-directory: web - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -31,223 +31,224 @@ jobs: - name: Build Vue app run: yarn run build - test-e2e-puppeteer: - runs-on: ubuntu-22.04 - services: - postgres: - image: postgres:latest - env: - POSTGRES_DB: django - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - rabbitmq: - image: rabbitmq:management - ports: - - 5672:5672 - minio: - # This image does not require any command arguments (which GitHub Actions don't support) - image: bitnami/minio:latest - env: - MINIO_ROOT_USER: minioAccessKey - MINIO_ROOT_PASSWORD: minioSecretKey - ports: - - 9000:9000 - env: - # API server env vars - DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django - DJANGO_MINIO_STORAGE_ENDPOINT: localhost:9000 - DJANGO_MINIO_STORAGE_ACCESS_KEY: minioAccessKey - DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey - DJANGO_STORAGE_BUCKET_NAME: dandi-bucket - DJANGO_DANDI_DANDISETS_BUCKET_NAME: dandi-bucket - DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME: dandiapi-dandisets-logs - DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME: dandiapi-embargo-dandisets-logs - DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 - DJANGO_DANDI_API_URL: http://localhost:8000 - DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/ - DJANGO_DANDI_DEV_EMAIL: test@example.com - DANDI_ALLOW_LOCALHOST_URLS: 1 - - # Web client env vars - VITE_APP_DANDI_API_ROOT: http://localhost:8000/api/ - VITE_APP_OAUTH_API_ROOT: http://localhost:8000/oauth/ - VITE_APP_OAUTH_CLIENT_ID: Dk0zosgt1GAAKfN8LT4STJmLJXwMDPbYWYzfNtAl - - # E2E tests env vars - CLIENT_URL: http://localhost:8085 - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - cache: 'yarn' - cache-dependency-path: web/yarn.lock - - - name: Install web app - if: steps.yarn-cache.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile --prefer-offline - working-directory: web - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install latest version of pip - run: pip install --upgrade pip - - - uses: actions/cache@v4 - id: pip-cache - with: - path: ${{ env.pythonLocation}}/lib/python3.11/site-packages/* - key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('setup.py') }} - - - name: Install dandi-api dependencies - run: pip install --upgrade --upgrade-strategy eager -e .[dev] - - - name: Apply migrations to API server - run: python manage.py migrate - - - name: Create any cache tables - run: python manage.py createcachetable - - # Populate the database with a dandiset for FileBrowser tests - - name: Create a superuser - run: python manage.py createsuperuser --noinput --email admin@example.com - - - name: Create Dandiset - run: python manage.py create_dev_dandiset --owner admin@example.com - - - name: Install E2E tests - run: yarn install --frozen-lockfile - working-directory: e2e/puppeteer - - - name: Lint E2E tests - run: yarn run lint --no-fix --max-warnings=0 - working-directory: e2e/puppeteer - - - name: Run E2E tests - run: | - # start vue dev server and wait for it to start - yarn --cwd ../../web/ run dev 2> /dev/null & - while ! nc -z localhost 8085; do - sleep 3 - done - - # start the dandi-api server - python ../../manage.py runserver & - - # run the E2E tests - yarn run test - working-directory: e2e/puppeteer - - test-e2e-playwright: - runs-on: ubuntu-22.04 - services: - postgres: - image: postgres:latest - env: - POSTGRES_DB: django - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - rabbitmq: - image: rabbitmq:management - ports: - - 5672:5672 - minio: - # This image does not require any command arguments (which GitHub Actions don't support) - image: bitnami/minio:latest - env: - MINIO_ROOT_USER: minioAccessKey - MINIO_ROOT_PASSWORD: minioSecretKey - ports: - - 9000:9000 - env: - # API server env vars - DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django - DJANGO_MINIO_STORAGE_ENDPOINT: localhost:9000 - DJANGO_MINIO_STORAGE_ACCESS_KEY: minioAccessKey - DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey - DJANGO_STORAGE_BUCKET_NAME: dandi-bucket - DJANGO_DANDI_DANDISETS_BUCKET_NAME: dandi-bucket - DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME: dandiapi-dandisets-logs - DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME: dandiapi-embargo-dandisets-logs - DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 - DJANGO_DANDI_API_URL: http://localhost:8000 - DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/ - DJANGO_DANDI_DEV_EMAIL: test@example.com - DANDI_ALLOW_LOCALHOST_URLS: 1 - - # Web client env vars - VITE_APP_DANDI_API_ROOT: http://localhost:8000/api/ - VITE_APP_OAUTH_API_ROOT: http://localhost:8000/oauth/ - VITE_APP_OAUTH_CLIENT_ID: Dk0zosgt1GAAKfN8LT4STJmLJXwMDPbYWYzfNtAl - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install web app - run: yarn install --frozen-lockfile - working-directory: web - - - name: Install latest version of pip - run: pip install --upgrade pip - - - uses: actions/cache@v4 - id: pip-cache - with: - path: ${{ env.pythonLocation}}/lib/python3.11/site-packages/* - key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('setup.py') }} - - - name: Install dandi-api dependencies - run: pip install --upgrade --upgrade-strategy eager -e .[dev] - - - name: Apply migrations to API server - run: python manage.py migrate - - - name: Create any cache tables - run: python manage.py createcachetable - - - name: Install test data - run: python manage.py loaddata playwright - - - name: Install dependencies - run: yarn install --frozen-lockfile - working-directory: e2e/playwright - - - name: Install Playwright Browsers - run: npx playwright install --with-deps - working-directory: e2e/playwright - - - name: Run Playwright tests - run: | - # start vue dev server - yarn --cwd ../../web/ run dev 2> /dev/null & - while ! nc -z localhost 8085; do - sleep 3 - done - - # start the dandi-api server - python ../../manage.py runserver & - - # run the tests - npx playwright test - working-directory: e2e/playwright - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: playwright-report - path: e2e/playwright/playwright-report/ - retention-days: 30 +#<<<<<<< HEAD +# test-e2e-puppeteer: +# runs-on: ubuntu-latest +# services: +# postgres: +# image: postgres:latest +# env: +# POSTGRES_DB: django +# POSTGRES_PASSWORD: postgres +# ports: +# - 5432:5432 +# rabbitmq: +# image: rabbitmq:management +# ports: +# - 5672:5672 +# minio: +# # This image does not require any command arguments (which GitHub Actions don't support) +# image: bitnami/minio:latest +# env: +# MINIO_ROOT_USER: minioAccessKey +# MINIO_ROOT_PASSWORD: minioSecretKey +# ports: +# - 9000:9000 +# env: +# # API server env vars +# DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django +# DJANGO_MINIO_STORAGE_ENDPOINT: localhost:9000 +# DJANGO_MINIO_STORAGE_ACCESS_KEY: minioAccessKey +# DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey +# DJANGO_STORAGE_BUCKET_NAME: dandi-bucket +# DJANGO_DANDI_DANDISETS_BUCKET_NAME: dandi-bucket +# DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME: dandiapi-dandisets-logs +# DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME: dandiapi-embargo-dandisets-logs +# DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 +# DJANGO_DANDI_API_URL: http://localhost:8000 +# DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/ +# DJANGO_DANDI_DEV_EMAIL: test@example.com +# DANDI_ALLOW_LOCALHOST_URLS: 1 +# +# # Web client env vars +# VITE_APP_DANDI_API_ROOT: http://localhost:8000/api/ +# VITE_APP_OAUTH_API_ROOT: http://localhost:8000/oauth/ +# VITE_APP_OAUTH_CLIENT_ID: Dk0zosgt1GAAKfN8LT4STJmLJXwMDPbYWYzfNtAl +# +# # E2E tests env vars +# CLIENT_URL: http://localhost:8085 +# +# steps: +# - uses: actions/checkout@v4 +# +# - uses: actions/setup-node@v4 +# with: +# cache: 'yarn' +# cache-dependency-path: web/yarn.lock +# +# - name: Install web app +# if: steps.yarn-cache.outputs.cache-hit != 'true' +# run: yarn install --frozen-lockfile --prefer-offline +# working-directory: web +# +# - name: Set up Python +# uses: actions/setup-python@v5 +# with: +# python-version: '3.11' +# +# - name: Install latest version of pip +# run: pip install --upgrade pip +# +# - uses: actions/cache@v4 +# id: pip-cache +# with: +# path: ${{ env.pythonLocation}}/lib/python3.11/site-packages/* +# key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('setup.py') }} +# +# - name: Install dandi-api dependencies +# run: pip install --upgrade --upgrade-strategy eager -e .[dev] +# +# - name: Apply migrations to API server +# run: python manage.py migrate +# +# - name: Create any cache tables +# run: python manage.py createcachetable +# +# # Populate the database with a dandiset for FileBrowser tests +# - name: Create a superuser +# run: python manage.py createsuperuser --noinput --email admin@example.com +# +# - name: Create Dandiset +# run: python manage.py create_dev_dandiset --owner admin@example.com +# +# - name: Install E2E tests +# run: yarn install --frozen-lockfile +# working-directory: e2e/puppeteer +# +# - name: Lint E2E tests +# run: yarn run lint --no-fix --max-warnings=0 +# working-directory: e2e/puppeteer +# +# - name: Run E2E tests +# run: | +# # start vue dev server and wait for it to start +# yarn --cwd ../../web/ run dev 2> /dev/null & +# while ! nc -z localhost 8085; do +# sleep 3 +# done +# +# # start the dandi-api server +# python ../../manage.py runserver & +# +# # run the E2E tests +# yarn run test +# working-directory: e2e/puppeteer +# +# test-e2e-playwright: +# runs-on: ubuntu-latest +# services: +# postgres: +# image: postgres:latest +# env: +# POSTGRES_DB: django +# POSTGRES_PASSWORD: postgres +# ports: +# - 5432:5432 +# rabbitmq: +# image: rabbitmq:management +# ports: +# - 5672:5672 +# minio: +# # This image does not require any command arguments (which GitHub Actions don't support) +# image: bitnami/minio:latest +# env: +# MINIO_ROOT_USER: minioAccessKey +# MINIO_ROOT_PASSWORD: minioSecretKey +# ports: +# - 9000:9000 +# env: +# # API server env vars +# DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django +# DJANGO_MINIO_STORAGE_ENDPOINT: localhost:9000 +# DJANGO_MINIO_STORAGE_ACCESS_KEY: minioAccessKey +# DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey +# DJANGO_STORAGE_BUCKET_NAME: dandi-bucket +# DJANGO_DANDI_DANDISETS_BUCKET_NAME: dandi-bucket +# DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME: dandiapi-dandisets-logs +# DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME: dandiapi-embargo-dandisets-logs +# DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 +# DJANGO_DANDI_API_URL: http://localhost:8000 +# DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/ +# DJANGO_DANDI_DEV_EMAIL: test@example.com +# DANDI_ALLOW_LOCALHOST_URLS: 1 +# +# # Web client env vars +# VITE_APP_DANDI_API_ROOT: http://localhost:8000/api/ +# VITE_APP_OAUTH_API_ROOT: http://localhost:8000/oauth/ +# VITE_APP_OAUTH_CLIENT_ID: Dk0zosgt1GAAKfN8LT4STJmLJXwMDPbYWYzfNtAl +# steps: +# - uses: actions/checkout@v4 +# +# - uses: actions/setup-node@v4 +# with: +# node-version: 18 +# +# - name: Set up Python +# uses: actions/setup-python@v5 +# with: +# python-version: '3.11' +# +# - name: Install web app +# run: yarn install --frozen-lockfile +# working-directory: web +# +# - name: Install latest version of pip +# run: pip install --upgrade pip +# +# - uses: actions/cache@v4 +# id: pip-cache +# with: +# path: ${{ env.pythonLocation}}/lib/python3.11/site-packages/* +# key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('setup.py') }} +# +# - name: Install dandi-api dependencies +# run: pip install --upgrade --upgrade-strategy eager -e .[dev] +# +# - name: Apply migrations to API server +# run: python manage.py migrate +# +# - name: Create any cache tables +# run: python manage.py createcachetable +# +# - name: Install test data +# run: python manage.py loaddata playwright +# +# - name: Install dependencies +# run: yarn install --frozen-lockfile +# working-directory: e2e/playwright +# +# - name: Install Playwright Browsers +# run: npx playwright install --with-deps +# working-directory: e2e/playwright +# +# - name: Run Playwright tests +# run: | +# # start vue dev server +# yarn --cwd ../../web/ run dev 2> /dev/null & +# while ! nc -z localhost 8085; do +# sleep 3 +# done +# +# # start the dandi-api server +# python ../../manage.py runserver & +# +# # run the tests +# npx playwright test +# working-directory: e2e/playwright +# +# - uses: actions/upload-artifact@v4 +# if: always() +# with: +# name: playwright-report +# path: e2e/playwright/playwright-report/ +# retention-days: 30 diff --git a/.github/workflows/pg-backup.yml b/.github/workflows/pg-backup.yml new file mode 100644 index 000000000..f1909ace4 --- /dev/null +++ b/.github/workflows/pg-backup.yml @@ -0,0 +1,63 @@ +name: Backup Production Heroku Database to S3 + +# This workflow invokes a Heroku "logical" backup: https://devcenter.heroku.com/articles/heroku-postgres-logical-backups +# +# Backups are already done via Heroku continuous protection services: https://devcenter.heroku.com/articles/heroku-postgres-data-safety-and-continuous-protection +# +# The reason behind this workflow is 1. a further safeguard for backing up our data, 2. the ability to use Postgres anywhere potentially, not just Heroku in the long-term if desired + +on: + workflow_dispatch: # Allows for manual trigger if ad-hoc is desired + schedule: + - cron: '0 5 * * *' # Runs at 5AM UTC every day -- which is also 12AM EST at MIT + +jobs: + backup: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Heroku CLI + run: | + curl https://cli-assets.heroku.com/install.sh | sh + + - name: Install AWS CLI (v2) + run: | + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install --update + aws --version # Check that AWS CLI is installed + + - name: Install Specific Version of PostgreSQL Client + run: | + sudo apt-get -y install wget ca-certificates + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + sudo apt-get update + sudo apt-get -y install postgresql-client-15 + + - name: Capture Backup + env: + HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} + run: | + DATABASE_URL=$(heroku config:get DATABASE_URL -a ${{ secrets.HEROKU_APP_NAME }}) + + # Parsing the DATABASE_URL to extract components + DB_USER=$(echo $DATABASE_URL | cut -d':' -f2 | sed 's|//||g') + DB_PASS=$(echo $DATABASE_URL | cut -d':' -f3 | cut -d'@' -f1) + DB_HOST=$(echo $DATABASE_URL | cut -d'@' -f2 | cut -d':' -f1) + DB_PORT=$(echo $DATABASE_URL | cut -d':' -f4 | cut -d'/' -f1) + DB_NAME=$(echo $DATABASE_URL | cut -d'/' -f4) + + export PGPASSWORD=$DB_PASS + + pg_dump -Fc --no-acl --no-owner -h $DB_HOST -U $DB_USER -d $DB_NAME -p $DB_PORT > db.dump + + - name: Upload Backup to S3 + run: | + aws s3 cp db.dump s3://${{ secrets.S3_BUCKET_NAME }}/$(date +%Y-%m-%d_%H-%M-%S)_db.dump + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: us-east-2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 055582c42..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Release - -on: - push: - branches: - - master - -jobs: - release: - if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')" - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # fetch history for all branches and tags - token: ${{ secrets.GH_TOKEN }} # use PAT with permissions to push to master - - - name: Download latest auto - run: | - auto_download_url="$(curl -fsSL https://api.github.com/repos/intuit/auto/releases/latest | jq -r '.assets[] | select(.name == "auto-linux.gz") | .browser_download_url')" - wget -O- "$auto_download_url" | gunzip > ~/auto - chmod a+x ~/auto - - - name: Create Release - env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - run: | - ~/auto shipit --message="auto shipit - CHANGELOG.md etc" diff --git a/.github/workflows_to_fix/auto-add-issues.yml b/.github/workflows_to_fix/auto-add-issues.yml new file mode 100644 index 000000000..1dc30f4c3 --- /dev/null +++ b/.github/workflows_to_fix/auto-add-issues.yml @@ -0,0 +1,17 @@ +#name: Add issues to beta project +# +#on: +# issues: +# types: +# - opened +# - transferred +# +#jobs: +# add-to-project: +# name: Add issue to project +# runs-on: ubuntu-latest +# steps: +# - uses: actions/add-to-project@v0.5.0 +# with: +# project-url: https://github.com/orgs/dandi/projects/16 +# github-token: ${{ secrets.AUTO_ADD_ISSUES }} diff --git a/.github/workflows_to_fix/frontend-ci.yml b/.github/workflows_to_fix/frontend-ci.yml new file mode 100644 index 000000000..178e04bfb --- /dev/null +++ b/.github/workflows_to_fix/frontend-ci.yml @@ -0,0 +1,137 @@ +name: CI for frontend +on: + pull_request: + push: + branches: + - master + schedule: + - cron: "0 0 * * *" +jobs: + lint-type-check: + defaults: + run: + working-directory: web + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 16 # TODO: use latest version when we update to Vue 3 + + - name: Install Vue app + run: yarn install + + - name: Lint Vue app + run: yarn run lint --no-fix + + - name: Type-check Vue app + run: yarn run type-check + + - name: Build Vue app + run: yarn run build + +# test-e2e: +# runs-on: ubuntu-latest +# services: +# postgres: +# image: postgres:latest +# env: +# POSTGRES_DB: django +# POSTGRES_PASSWORD: postgres +# ports: +# - 5432:5432 +# rabbitmq: +# image: rabbitmq:management +# ports: +# - 5672:5672 +# minio: +# # This image does not require any command arguments (which GitHub Actions don't support) +# image: bitnami/minio:latest +# env: +# MINIO_ROOT_USER: minioAccessKey +# MINIO_ROOT_PASSWORD: minioSecretKey +# ports: +# - 9000:9000 +# env: +# # API server env vars +# DJANGO_DATABASE_URL: postgres://postgres:postgres@localhost:5432/django +# DJANGO_MINIO_STORAGE_ENDPOINT: localhost:9000 +# DJANGO_MINIO_STORAGE_ACCESS_KEY: minioAccessKey +# DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey +# DJANGO_STORAGE_BUCKET_NAME: dandi-bucket +# DJANGO_DANDI_DANDISETS_BUCKET_NAME: dandi-bucket +# DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME: dandiapi-dandisets-logs +# DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME: dandi-embargo-dandisets +# DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME: dandiapi-embargo-dandisets-logs +# DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 +# DJANGO_DANDI_API_URL: http://localhost:8000 +# DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org/ +# DANDI_ALLOW_LOCALHOST_URLS: 1 +# +# # Web client env vars +# VITE_APP_DANDI_API_ROOT: http://localhost:8000/api/ +# VITE_APP_OAUTH_API_ROOT: http://localhost:8000/oauth/ +# VITE_APP_OAUTH_CLIENT_ID: Dk0zosgt1GAAKfN8LT4STJmLJXwMDPbYWYzfNtAl +# +# # E2E tests env vars +# CLIENT_URL: http://localhost:8085 +# +# steps: +# - uses: actions/checkout@v4 +# +# - uses: actions/setup-node@v4 +# with: +# cache: 'yarn' +# cache-dependency-path: web/yarn.lock +# +# - name: Install web app +# if: steps.yarn-cache.outputs.cache-hit != 'true' +# run: yarn install --frozen-lockfile --prefer-offline +# working-directory: web +# +# - name: Set up Python +# uses: actions/setup-python@v5 +# with: +# python-version: '3.11' +# +# - name: Install latest version of pip +# run: pip install --upgrade pip +# +# - uses: actions/cache@v3 +# id: pip-cache +# with: +# path: ${{ env.pythonLocation}}/lib/python3.11/site-packages/* +# key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('setup.py') }} +# +# - name: Install dandi-api dependencies +# run: pip install --upgrade --upgrade-strategy eager -e .[dev] +# +# - name: Apply migrations to API server +# run: python manage.py migrate +# +# - name: Create any cache tables +# run: python manage.py createcachetable +# +# - name: Install E2E tests +# run: yarn install --frozen-lockfile +# working-directory: e2e +# +# - name: Lint E2E tests +# run: yarn run lint --no-fix --max-warnings=0 +# working-directory: e2e +# +# - name: Run E2E tests +# run: | +# # start vue dev server and wait for it to start +# yarn --cwd ../web/ run dev 2> /dev/null & +# while ! nc -z localhost 8085; do +# sleep 3 +# done +# +# # start the dandi-api server +# python ../manage.py runserver & +# +# # run the E2E tests +# yarn run test +# working-directory: e2e diff --git a/.github/workflows_to_fix/release.yml b/.github/workflows_to_fix/release.yml new file mode 100644 index 000000000..156958bf8 --- /dev/null +++ b/.github/workflows_to_fix/release.yml @@ -0,0 +1,28 @@ +#name: Release +# +#on: +# push: +# branches: +# - master +# +#jobs: +# release: +# if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')" +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# with: +# fetch-depth: 0 # fetch history for all branches and tags +# token: ${{ secrets.GH_TOKEN }} # use PAT with permissions to push to master +# +# - name: Download latest auto +# run: | +# auto_download_url="$(curl -fsSL https://api.github.com/repos/intuit/auto/releases/latest | jq -r '.assets[] | select(.name == "auto-linux.gz") | .browser_download_url')" +# wget -O- "$auto_download_url" | gunzip > ~/auto +# chmod a+x ~/auto +# +# - name: Create Release +# env: +# GH_TOKEN: ${{ secrets.GH_TOKEN }} +# run: | +# ~/auto shipit --message="auto shipit - CHANGELOG.md etc" diff --git a/.gitignore b/.gitignore index 892b644e8..09d6c4d00 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ local_settings.py db.sqlite3 db.sqlite3-journal media +.idea/ # If your build process includes running collectstatic, then you probably don't need or want to include staticfiles/ # in your Git repository. Update and uncomment the following line accordingly. @@ -125,5 +126,8 @@ dmypy.json # Editor settings .vscode - +<<<<<<< HEAD +======= +.idea/ +>>>>>>> upstream/master .DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ea8c66ee..3325e91e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,102 @@ +# v0.3.115 (Mon Dec 09 2024) + +#### 🐛 Bug Fix + +- Bump `django-allauth` to latest version [#2099](https://github.com/dandi/dandi-archive/pull/2099) ([@mvandenburgh](https://github.com/mvandenburgh)) + +#### Authors: 1 + +- Mike VanDenburgh ([@mvandenburgh](https://github.com/mvandenburgh)) + +--- + +# v0.3.114 (Mon Dec 09 2024) + +#### 🐛 Bug Fix + +- Pin `django-allauth` to 0.61.1 [#2098](https://github.com/dandi/dandi-archive/pull/2098) ([@mvandenburgh](https://github.com/mvandenburgh)) +- Set `--max-warnings` to zero for `eslint` [#2088](https://github.com/dandi/dandi-archive/pull/2088) ([@mvandenburgh](https://github.com/mvandenburgh)) + +#### Authors: 1 + +- Mike VanDenburgh ([@mvandenburgh](https://github.com/mvandenburgh)) + +--- + +# v0.3.113 (Wed Dec 04 2024) + +#### 🐛 Bug Fix + +- Fix validation error when only Zarr assets are uploaded [#2062](https://github.com/dandi/dandi-archive/pull/2062) ([@jjnesbitt](https://github.com/jjnesbitt) [@aaronkanzer](https://github.com/aaronkanzer)) + +#### Authors: 2 + +- [@aaronkanzer](https://github.com/aaronkanzer) +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) + +--- + +# v0.3.112 (Mon Dec 02 2024) + +#### 🐛 Bug Fix + +- Add incomplete upload dialog to DLP when unembargo is blocked [#2082](https://github.com/dandi/dandi-archive/pull/2082) ([@jjnesbitt](https://github.com/jjnesbitt)) + +#### Authors: 1 + +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) + +--- + +# v0.3.111 (Mon Dec 02 2024) + +:tada: This release contains work from a new contributor! :tada: + +Thank you, null[@aaronkanzer](https://github.com/aaronkanzer), for all your work! + +#### 🐛 Bug Fix + +- Empty PR to trigger release [#2086](https://github.com/dandi/dandi-archive/pull/2086) ([@jjnesbitt](https://github.com/jjnesbitt)) +- Include robots.txt in UI and API for handling of web crawlers [#2084](https://github.com/dandi/dandi-archive/pull/2084) (aaronkanzer@Aarons-MacBook-Pro.local [@jjnesbitt](https://github.com/jjnesbitt) [@aaronkanzer](https://github.com/aaronkanzer)) +- Use a dedicated logger. not top level logging. module [#2077](https://github.com/dandi/dandi-archive/pull/2077) ([@yarikoptic](https://github.com/yarikoptic)) +- Add API support for Embargoed Zarrs [#2069](https://github.com/dandi/dandi-archive/pull/2069) ([@jjnesbitt](https://github.com/jjnesbitt)) + +#### Authors: 4 + +- [@aaronkanzer](https://github.com/aaronkanzer) +- Aaron Kanzer (aaronkanzer@Aarons-MacBook-Pro.local) +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + +# v0.3.110 (Wed Nov 13 2024) + +#### 🐛 Bug Fix + +- Fix display of embargoed dandiset error page in GUI [#2073](https://github.com/dandi/dandi-archive/pull/2073) ([@jjnesbitt](https://github.com/jjnesbitt)) + +#### Authors: 1 + +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) + +--- + +# v0.3.109 (Thu Oct 31 2024) + +#### 🐛 Bug Fix + +- Add neurosift external service for dandisets [#2041](https://github.com/dandi/dandi-archive/pull/2041) ([@magland](https://github.com/magland) [@waxlamp](https://github.com/waxlamp)) +- Display message in GUI when accessing embargoed dandiset [#2060](https://github.com/dandi/dandi-archive/pull/2060) ([@jjnesbitt](https://github.com/jjnesbitt)) + +#### Authors: 3 + +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) +- Jeremy Magland ([@magland](https://github.com/magland)) +- Roni Choudhury ([@waxlamp](https://github.com/waxlamp)) + +--- + # v0.3.108 (Thu Oct 24 2024) #### 🐛 Bug Fix diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3484b8c9d..15fc551be 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -3,26 +3,19 @@ ## Develop with Docker (recommended quickstart) This is the simplest configuration for developers to start with. -### Initial Setup -1. Run `docker compose run --rm django ./manage.py migrate` -2. Run `docker compose run --rm django ./manage.py createcachetable` -3. Run `docker compose run --rm django ./manage.py createsuperuser --email $(git config user.email)` - and follow the prompts to create your own user. - This sets your username to your git email to ensure parity with how GitHub logins work. You can also replace the command substitution expression with a literal email address, or omit the `--email` option entirely to run the command in interactive mode. -4. Run `docker compose run --rm django ./manage.py create_dev_dandiset --owner $(git config user.email)` - to create a dummy dandiset to start working with. +## Instructions for local development with front-end hot reloading -### Run Application -1. Run `docker compose up` -2. Access the site, starting at http://localhost:8000/admin/ -3. When finished, use `Ctrl+C` +1. Ensure you have installed Docker on your local machine +2. Run `./admin_dev_startup.sh `. When prompted, enter an username and password in the command prompt. (If you run into local errors with the script, you may need to run `chmod +x admin_dev_startup.sh` first) +3. Navigate to `localhost:8000/admin`, and log in with the username and password you used in Step #2. Under the `User` section, select the username and change the `Status` from `Incomplete` to `Approved`. +4. Navigate to `localhost:8085` and select `LOG IN WITH GITHUB`. ### Application Maintenance Occasionally, new package dependencies or schema changes will necessitate maintenance. To non-destructively update your development stack at any time: -1. Run `docker compose pull` -2. Run `docker compose build --pull --no-cache` -3. Run `docker compose run --rm django ./manage.py migrate` +1. Run `docker-compose pull` +2. Run `docker-compose build --pull --no-cache` +3. Run `docker-compose run --rm django ./manage.py migrate` ## Develop Natively (advanced) This configuration still uses Docker to run attached services in the background, @@ -30,7 +23,7 @@ but allows developers to run Python code on their native system. ### Initial Setup 1. Install [Docker](https://docs.docker.com/engine/install/) and [Docker Compose](https://docs.docker.com/compose/install/) -2. Run `docker compose -f ./docker-compose.yml up -d` +2. Run `docker-compose -f ./docker-compose.yml up -d` 3. Install Python 3.11 4. Install [`psycopg2` build prerequisites](https://www.psycopg.org/docs/install.html#build-prerequisites). @@ -50,20 +43,20 @@ but allows developers to run Python code on their native system. to create a dummy dandiset to start working with. ### Run Application -1. Ensure `docker compose -f ./docker-compose.yml up -d` is still active +1. Ensure `docker-compose -f ./docker-compose.yml up -d` is still active 2. Run: 1. `source ./dev/export-env.sh` 2. `./manage.py runserver` 3. Run in a separate terminal: 1. `source ./dev/export-env.sh` 2. `celery --app dandiapi.celery worker --loglevel INFO --without-heartbeat -Q celery,calculate_sha256,ingest_zarr_archive -B` -4. When finished, run `docker compose stop` +4. When finished, run `docker-compose stop` ## Remap Service Ports (optional) Attached services may be exposed to the host system via alternative ports. Developers who work on multiple software projects concurrently may find this helpful to avoid port conflicts. -To do so, before running any `docker compose` commands, set any of the environment variables: +To do so, before running any `docker-compose` commands, set any of the environment variables: * `DOCKER_POSTGRES_PORT` * `DOCKER_RABBITMQ_PORT` * `DOCKER_MINIO_PORT` @@ -87,7 +80,7 @@ To install the tox pytest dependencies into your environment, run `pip install - These are useful for IDE autocompletion or if you want to run `pytest` directly (not recommended). When running the "Develop with Docker" configuration, all tox commands must be run as -`docker compose run --rm django tox`; extra arguments may also be appended to this form. +`docker-compose run --rm django tox`; extra arguments may also be appended to this form. ### Running Tests Run `tox` to launch the full test suite. diff --git a/README.md b/README.md index 49d3b9c3d..6b6abc7a2 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,25 @@ -# DANDI Archive +# LINC Data Platform -![](https://www.dandiarchive.org/assets/dandi_logo.svg) +![LINC Logo](web/src/assets/linc.logo.color+white.png) -## DANDI: Distributed Archives for Neurophysiology Data Integration +LINC Data Platform is data sharing and visualization platform for the center for Large-scale Imaging of Neural Circuits (LINC). [See here for details](https://connects.mgh.harvard.edu/) -[DANDI](https://dandiarchive.org/) is a platform for publishing, sharing, and processing neurophysiology data -funded by the [BRAIN Initiative](https://braininitiative.nih.gov/). The archive -accepts cellular neurophysiology data including electrophysiology, -optophysiology, and behavioral time-series, and images from immunostaining -experiments. This archive is not just an endpoint to store data, it is intended as a living repository that enables -collaboration within and across labs, as well as the entry point for research. +The LINC Data Platform site can be (visited online here)[https://lincbrain.org] + +This repository is a fork of the [DANDI Archive](https://github.com/dandi/dandi-archive) project. For more information, please visit the [DANDI Archive](https://dandiarchive.org/) ## Structure -The dandi-archive repository contains a Django-based [backend](dandiapi/) to run the DANDI REST API, and a +The linc-archive repository contains a Django-based [backend](dandiapi/) to run the LINC REST API, and a Vue-based [frontend](web/) to provide a user interface to the archive. ## Resources -* To learn how to interact with the archive, -see [the handbook](https://www.dandiarchive.org/handbook/). +* To learn how to interact with LINC Data Platform, please refer to the [LINC Documentation](https://docs.lincbrain.org/). * To get help: - - ask a question: https://github.com/dandi/helpdesk/discussions - - file a feature request or bug report: https://github.com/dandi/helpdesk/issues/new/choose - - contact the DANDI team: help@dandiarchive.org + - File an issue on the relevant [GitHub repository](https://github.com/lincbrain) + - Reach out on the [LINC Slack](https://mit-lincbrain.slack.com/) * To understand how to hack on the archive codebase: - Django backend: [`DEVELOPMENT.md`](DEVELOPMENT.md) diff --git a/admin_dev_startup.sh b/admin_dev_startup.sh new file mode 100755 index 000000000..347392f52 --- /dev/null +++ b/admin_dev_startup.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# This script can be used to launch the front-end and back-end applications locally. +# Happy developing! + +if [ $# -lt 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +image_name=$1 +email=$2 + +cd web/ + +# Build Docker image (include the path to the Dockerfile's context) +docker build -t $image_name -f Dockerfile.dev . + +# Run the Docker container for frontend in background +docker run -d -v "$(pwd):/usr/src/app" -v /usr/src/app/node_modules -p 8085:8085 -e CHOKIDAR_USEPOLLING=true $image_name + +cd .. + +# Run Docker Compose commands for backend +docker compose run --rm django ./manage.py migrate +docker compose run --rm django ./manage.py createcachetable +docker compose run --rm django ./manage.py createsuperuser +docker compose run --rm django ./manage.py create_dev_dandiset --owner $email + +# Bring backend application to life! +docker compose up diff --git a/dandiapi/api/admin.py b/dandiapi/api/admin.py index f7fbbf62f..ae2a8939c 100644 --- a/dandiapi/api/admin.py +++ b/dandiapi/api/admin.py @@ -24,6 +24,9 @@ Upload, UserMetadata, Version, + WebKnossosAnnotation, + WebKnossosDataset, + WebKnossosDataLayer ) from dandiapi.api.views.users import social_account_to_dict from dandiapi.zarr.tasks import ingest_dandiset_zarrs @@ -37,12 +40,23 @@ class UserMetadataInline(TabularInline): model = UserMetadata - fields = ['status', 'questionnaire_form', 'rejection_reason'] - + fields = ['status', 'webknossos_credential', 'questionnaire_form', 'rejection_reason'] class SocialAccountInline(TabularInline): model = SocialAccount +@admin.register(WebKnossosAnnotation) +class WebKnossosAnnotationAdmin(admin.ModelAdmin): + model = WebKnossosAnnotation + +@admin.register(WebKnossosDataset) +class WebKnossosDatasetAdmin(admin.ModelAdmin): + model = WebKnossosDataset + +@admin.register(WebKnossosDataLayer) +class WebKnossosDataLayerAdmin(admin.ModelAdmin): + model = WebKnossosDataLayer + class UserAdmin(BaseUserAdmin): list_select_related = ['metadata'] diff --git a/dandiapi/api/doi.py b/dandiapi/api/doi.py index 92f272d54..685e383e3 100644 --- a/dandiapi/api/doi.py +++ b/dandiapi/api/doi.py @@ -17,6 +17,8 @@ (settings.DANDI_DOI_API_PREFIX, 'DANDI_DOI_API_PREFIX'), ] +logger = logging.getLogger(__name__) + def doi_configured() -> bool: return any(setting is not None for setting, _ in DANDI_DOI_SETTINGS) @@ -51,10 +53,10 @@ def create_doi(version: Version) -> str: timeout=30, ).raise_for_status() except requests.exceptions.HTTPError as e: - logging.exception('Failed to create DOI %s', doi) - logging.exception(request_body) + logger.exception('Failed to create DOI %s', doi) + logger.exception(request_body) if e.response: - logging.exception(e.response.text) + logger.exception(e.response.text) raise return doi @@ -70,13 +72,13 @@ def delete_doi(doi: str) -> None: r.raise_for_status() except requests.exceptions.HTTPError as e: if e.response and e.response.status_code == requests.codes.not_found: - logging.warning('Tried to get data for nonexistent DOI %s', doi) + logger.warning('Tried to get data for nonexistent DOI %s', doi) return - logging.exception('Failed to fetch data for DOI %s', doi) + logger.exception('Failed to fetch data for DOI %s', doi) raise if r.json()['data']['attributes']['state'] == 'draft': try: s.delete(doi_url).raise_for_status() except requests.exceptions.HTTPError: - logging.exception('Failed to delete DOI %s', doi) + logger.exception('Failed to delete DOI %s', doi) raise diff --git a/dandiapi/api/mail.py b/dandiapi/api/mail.py index 7b0a5b4d1..e9079d882 100644 --- a/dandiapi/api/mail.py +++ b/dandiapi/api/mail.py @@ -23,8 +23,8 @@ 'dandi_web_app_url': settings.DANDI_WEB_APP_URL, } -# TODO: turn this into a Django setting -ADMIN_EMAIL = 'info@dandiarchive.org' + +ADMIN_EMAIL = 'admin@lincbrain.org' def user_greeting_name(user: User, socialaccount: SocialAccount = None) -> str: @@ -68,7 +68,7 @@ def build_removed_message(dandiset, removed_owner): } # Email sent when a user is removed as an owner from a dandiset return build_message( - subject=f'Removed from Dandiset "{dandiset.draft_version.name}"', + subject=f'Removed from dataset "{dandiset.draft_version.name}"', message=render_to_string('api/mail/removed_message.txt', render_context), to=[removed_owner.email], ) @@ -82,7 +82,7 @@ def build_added_message(dandiset, added_owner): } # Email sent when a user is added as an owner of a dandiset return build_message( - subject=f'Added to Dandiset "{dandiset.draft_version.name}"', + subject=f'Added to dataset "{dandiset.draft_version.name}"', message=render_to_string('api/mail/added_message.txt', render_context), to=[added_owner.email], ) @@ -98,7 +98,7 @@ def send_ownership_change_emails(dandiset, removed_owners, added_owners): def build_registered_message(user: User, socialaccount: SocialAccount): # Email sent to the DANDI list when a new user logs in for the first time return build_message( - subject=f'DANDI: New user registered: {user.email}', + subject=f'LINC: New user registered: {user.email}', message=render_to_string( 'api/mail/registered_message.txt', {'greeting_name': user_greeting_name(user, socialaccount)}, @@ -121,7 +121,7 @@ def build_new_user_messsage(user: User, socialaccount: SocialAccount = None): } # Email sent to the DANDI list when a new user logs in for the first time return build_message( - subject=f'DANDI: Review new user: {user.username}', + subject=f'LINC Data Platform: Review new user: {user.username}', message=render_to_string('api/mail/new_user_message.txt', render_context), to=[ADMIN_EMAIL], ) @@ -136,7 +136,7 @@ def send_new_user_message_email(user: User, socialaccount: SocialAccount): def build_approved_user_message(user: User, socialaccount: SocialAccount = None): return build_message( - subject='Your DANDI Account', + subject='Your LINC Data Platform Account', message=render_to_string( 'api/mail/approved_user_message.txt', { @@ -157,7 +157,7 @@ def send_approved_user_message(user: User, socialaccount: SocialAccount): def build_rejected_user_message(user: User, socialaccount: SocialAccount = None): return build_message( - subject='Your DANDI Account', + subject='Your LINC Data Platform Account', message=render_to_string( 'api/mail/rejected_user_message.txt', { @@ -179,7 +179,7 @@ def send_rejected_user_message(user: User, socialaccount: SocialAccount): def build_pending_users_message(users: Iterable[User]): render_context = {**BASE_RENDER_CONTEXT, 'users': users} return build_message( - subject='DANDI: new user registrations to review', + subject='LINC Data Platform: new user registrations to review', message=render_to_string('api/mail/pending_users_message.txt', render_context), to=[ADMIN_EMAIL], ) @@ -192,6 +192,31 @@ def send_pending_users_message(users: Iterable[User]): connection.send_messages(messages) +def build_dandisets_to_unembargo_message(dandisets: Iterable[Dandiset]): + dandiset_context = [ + { + 'identifier': ds.identifier, + 'owners': [user.username for user in ds.owners], + 'asset_count': ds.draft_version.asset_count, + 'size': ds.draft_version.size, + } + for ds in dandisets + ] + render_context = {**BASE_RENDER_CONTEXT, 'dandisets': dandiset_context} + return build_message( + subject='DANDI: New Dandisets to unembargo', + message=render_to_string('api/mail/dandisets_to_unembargo.txt', render_context), + to=[settings.DANDI_DEV_EMAIL], + ) + + +def send_dandisets_to_unembargo_message(dandisets: Iterable[Dandiset]): + logger.info('Sending dandisets to unembargo message to devs at %s', settings.DANDI_DEV_EMAIL) + messages = [build_dandisets_to_unembargo_message(dandisets)] + with mail.get_connection() as connection: + connection.send_messages(messages) + + def build_dandiset_unembargoed_message(dandiset: Dandiset): dandiset_context = { 'identifier': dandiset.identifier, diff --git a/dandiapi/api/management/commands/create_dev_dandiset.py b/dandiapi/api/management/commands/create_dev_dandiset.py index a2819c2cb..4094fca27 100644 --- a/dandiapi/api/management/commands/create_dev_dandiset.py +++ b/dandiapi/api/management/commands/create_dev_dandiset.py @@ -1,4 +1,5 @@ from __future__ import annotations +import hashlib from uuid import uuid4 @@ -7,7 +8,7 @@ from django.core.files.uploadedfile import SimpleUploadedFile import djclick as click -from dandiapi.api.models import AssetBlob +from dandiapi.api.models import AssetBlob, WebKnossosDataset, WebKnossosAnnotation, WebKnossosDataLayer from dandiapi.api.services.asset import add_asset_to_version from dandiapi.api.services.dandiset import create_dandiset from dandiapi.api.services.metadata import validate_asset_metadata, validate_version_metadata @@ -17,8 +18,14 @@ @click.command() @click.option('--name', default='Development Dandiset') @click.option('--owner', 'email', required=True, help='The email address of the owner') -def create_dev_dandiset(*, name: str, email: str): +@click.option('--first_name', default='Randi The Admin') +@click.option('--last_name', default='Dandi') +def create_dev_dandiset(name: str, email: str, first_name: str, last_name: str): owner = User.objects.get(email=email) + owner.first_name = first_name + owner.last_name = last_name + owner.save() + version_metadata = { 'description': 'An informative description', @@ -28,28 +35,112 @@ def create_dev_dandiset(*, name: str, email: str): user=owner, embargo=False, version_name=name, version_metadata=version_metadata ) - uploaded_file = SimpleUploadedFile(name='foo/bar.txt', content=b'A' * 20) - etag = '76d36e98f312e98ff908c8c82c8dd623-0' - try: - asset_blob = AssetBlob.objects.get(etag=etag) - except AssetBlob.DoesNotExist: - asset_blob = AssetBlob( - blob_id=uuid4(), - blob=uploaded_file, - etag=etag, - size=20, + files_names_and_etags = [ + {"etag": "76d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/bar.txt"}, + {"etag": "86d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/buzz.txt"}, + {"etag": "a6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file2.txt"}, + {"etag": "b6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file3.txt"}, + {"etag": "c6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file4.txt"}, + {"etag": "d6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file5.txt"}, + {"etag": "e6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file6.txt"}, + {"etag": "f6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file7.txt"}, + {"etag": "g6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file8.txt"}, + {"etag": "h6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file9.txt"}, + {"etag": "i6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file10.txt"}, + {"etag": "j6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file11.txt"}, + {"etag": "k6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file12.txt"}, + {"etag": "l6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file13.txt"}, + {"etag": "m6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file14.txt"}, + {"etag": "n6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file15.txt"}, + {"etag": "o6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file16.txt"}, + {"etag": "p6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file17.txt"}, + {"etag": "q6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file18.txt"}, + {"etag": "r6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file19.txt"}, + {"etag": "s6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file20.txt"}, + {"etag": "t6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file21.txt"}, + {"etag": "u6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file22.txt"}, + {"etag": "v6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file23.txt"}, + {"etag": "w6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file24.txt"}, + {"etag": "x6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file25.txt"}, + {"etag": "y6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file26.txt"}, + {"etag": "z6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file27.txt"}, + {"etag": "06d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file28.txt"}, + {"etag": "16d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file29.txt"}, + {"etag": "26d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file30.txt"}, + {"etag": "36d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file31.txt"}, + {"etag": "46d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file32.txt"}, + {"etag": "56d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file33.txt"}, + {"etag": "66d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file34.txt"}, + {"etag": "76d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file35.txt"}, + {"etag": "86d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file36.txt"}, + {"etag": "96d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file37.txt"}, + {"etag": "a6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file38.txt"}, + {"etag": "b6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file39.txt"}, + {"etag": "c6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file40.txt"}, + {"etag": "d6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file41.txt"}, + {"etag": "e6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file42.txt"}, + {"etag": "f6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file43.txt"}, + {"etag": "g6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file44.txt"}, + {"etag": "h6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file45.txt"}, + {"etag": "i6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file46.txt"}, + {"etag": "j6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file47.txt"}, + {"etag": "k6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file48.txt"}, + {"etag": "l6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file49.txt"}, + {"etag": "m6d36e98f312e98ff908c8c82c8dd623-0", "file_name": "foo/file50.txt"}, + ] + + for index, file_name_and_etag in enumerate(files_names_and_etags): + file_size = 20 + file_content = b'A' * file_size + uploaded_file = SimpleUploadedFile( + name=file_name_and_etag["file_name"], + content=file_content ) - asset_blob.save() - asset_metadata = { - 'schemaVersion': settings.DANDI_SCHEMA_VERSION, - 'encodingFormat': 'text/plain', - 'schemaKey': 'Asset', - 'path': 'foo/bar.txt', - } - asset = add_asset_to_version( - user=owner, version=draft_version, asset_blob=asset_blob, metadata=asset_metadata - ) + etag = file_name_and_etag["etag"] + + try: + asset_blob = AssetBlob.objects.get(etag=etag) + except AssetBlob.DoesNotExist: + # Since the SimpleUploadedFile is non-zarr asset, validation fails + # without a sha2_256 initially provided. + sha256_hash = hashlib.sha256(file_content).hexdigest() + asset_blob = AssetBlob( + blob_id=uuid4(), blob=uploaded_file, etag=etag, size=file_size, sha256=sha256_hash + ) + asset_blob.save() + + asset_metadata = { + 'schemaVersion': settings.DANDI_SCHEMA_VERSION, + 'encodingFormat': 'text/plain', + 'schemaKey': 'Asset', + 'path': file_name_and_etag["file_name"], + } + asset = add_asset_to_version( + user=owner, version=draft_version, asset_blob=asset_blob, metadata=asset_metadata + ) + + # Create WebKnossosDataset for every other asset (e.g., even indices) + if index % 2 == 0: + webknossos_dataset = WebKnossosDataset.objects.create( + webknossos_dataset_name=file_name_and_etag["file_name"], + webknossos_organization_name="LINC" + ) + WebKnossosDataLayer.objects.create( + webknossos_dataset=webknossos_dataset, + asset=asset + ) + + for annotation_num in range(2): + WebKnossosAnnotation.objects.create( + webknossos_annotation_id=f"Annotation ID: {annotation_num + 1}", + webknossos_annotation_name=f"Annotation Name: {annotation_num + 1}", + webknossos_organization="LINC", + webknossos_annotation_owner_first_name="Randi", + webknossos_annotation_owner_last_name="Dandi", + webknossos_dataset=webknossos_dataset + ) + - calculate_sha256(blob_id=asset_blob.blob_id) - validate_asset_metadata(asset=asset) - validate_version_metadata(version=draft_version) + calculate_sha256(blob_id=asset_blob.blob_id) + validate_asset_metadata(asset=asset) + validate_version_metadata(version=draft_version) diff --git a/dandiapi/api/management/commands/createsuperuser.py b/dandiapi/api/management/commands/createsuperuser.py index e21975112..117f9d450 100644 --- a/dandiapi/api/management/commands/createsuperuser.py +++ b/dandiapi/api/management/commands/createsuperuser.py @@ -4,16 +4,24 @@ from composed_configuration._allauth_support.management.commands import createsuperuser from django.db.models.signals import post_save - +from django.db import IntegrityError from dandiapi.api.models.user import UserMetadata if TYPE_CHECKING: from composed_configuration._allauth_support.createsuperuser import EmailAsUsernameProxyUser - -def create_usermetadata(instance: EmailAsUsernameProxyUser, *args, **kwargs): - UserMetadata.objects.create(user=instance, status=UserMetadata.Status.APPROVED) - +def create_usermetadata(sender, instance, signal, **kwargs): + try: + UserMetadata.objects.get(user=instance) + except UserMetadata.DoesNotExist: + try: + UserMetadata.objects.create(user=instance, status=UserMetadata.Status.APPROVED) + except IntegrityError: + # Handle duplicate key issue gracefully + user_metadata = UserMetadata.objects.filter(user=instance).first() + if user_metadata: + user_metadata.status = UserMetadata.Status.APPROVED + user_metadata.save() class Command(createsuperuser.Command): def handle(self, *args, **kwargs) -> str | None: @@ -22,12 +30,13 @@ def handle(self, *args, **kwargs) -> str | None: # when this management command is actually run. post_save.connect(create_usermetadata, sender=createsuperuser.user_model) - # Save the return value of the parent class function so we can return it later - return_value = super().handle(*args, **kwargs) - - # Disconnect the signal handler. This isn't strictly necessary, but this avoids any - # unexpected behavior if, for example, someone extends this command and doesn't - # realize there's a signal handler attached dynamically. - post_save.disconnect(create_usermetadata, sender=createsuperuser.user_model) + try: + # Save the return value of the parent class function so we can return it later + return_value = super().handle(*args, **kwargs) + finally: + # Disconnect the signal handler. This isn't strictly necessary, but this avoids any + # unexpected behavior if, for example, someone extends this command and doesn't + # realize there's a signal handler attached dynamically. + post_save.disconnect(create_usermetadata, sender=createsuperuser.user_model) return return_value diff --git a/dandiapi/api/migrations/0010_auditrecord.py b/dandiapi/api/migrations/0010_auditrecord.py deleted file mode 100644 index a58f12eb3..000000000 --- a/dandiapi/api/migrations/0010_auditrecord.py +++ /dev/null @@ -1,51 +0,0 @@ -# Generated by Django 4.1.13 on 2024-07-24 01:30 -from __future__ import annotations - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ('api', '0009_remove_embargoedassetblob_dandiset_and_more'), - ] - - operations = [ - migrations.CreateModel( - name='AuditRecord', - fields=[ - ( - 'id', - models.BigAutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name='ID' - ), - ), - ('timestamp', models.DateTimeField(auto_now_add=True)), - ('dandiset_id', models.IntegerField()), - ('username', models.CharField(max_length=39)), - ('user_email', models.CharField(max_length=254)), - ('user_fullname', models.CharField(max_length=301)), - ( - 'record_type', - models.CharField( - choices=[ - ('create_dandiset', 'create_dandiset'), - ('change_owners', 'change_owners'), - ('update_metadata', 'update_metadata'), - ('add_asset', 'add_asset'), - ('update_asset', 'update_asset'), - ('remove_asset', 'remove_asset'), - ('create_zarr', 'create_zarr'), - ('upload_zarr_chunks', 'upload_zarr_chunks'), - ('delete_zarr_chunks', 'delete_zarr_chunks'), - ('finalize_zarr', 'finalize_zarr'), - ('unembargo_dandiset', 'unembargo_dandiset'), - ('publish_dandiset', 'publish_dandiset'), - ('delete_dandiset', 'delete_dandiset'), - ], - max_length=32, - ), - ), - ('details', models.JSONField(blank=True)), - ], - ), - ] diff --git a/dandiapi/api/migrations/0010_usermetadata_webknossos_credential.py b/dandiapi/api/migrations/0010_usermetadata_webknossos_credential.py new file mode 100644 index 000000000..f74a03680 --- /dev/null +++ b/dandiapi/api/migrations/0010_usermetadata_webknossos_credential.py @@ -0,0 +1,19 @@ +# Generated by Django 4.1.13 on 2024-07-07 20:55 +from __future__ import annotations + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0009_remove_embargoedassetblob_dandiset_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='usermetadata', + name='webknossos_credential', + field=models.CharField(blank=True, max_length=128, null=True), + ), + ] diff --git a/dandiapi/api/migrations/0011_asset_access_metadata.py b/dandiapi/api/migrations/0011_asset_access_metadata.py deleted file mode 100644 index 534880912..000000000 --- a/dandiapi/api/migrations/0011_asset_access_metadata.py +++ /dev/null @@ -1,81 +0,0 @@ -# Generated by Django 4.1.13 on 2024-08-20 17:21 -from __future__ import annotations - -from django.db import migrations, models -from django.db.models.expressions import RawSQL - - -def remove_access_fields(apps, _): - Asset = apps.get_model('api.Asset') - AuditRecord = apps.get_model('api.AuditRecord') - - # Use the postgres jsonb '-' operator to delete the 'access' field from metadata - Asset.objects.filter(published=False, metadata__access__isnull=False).update( - metadata=RawSQL("metadata - 'access'", []) - ) - - # Delete access field from existing audit records - # https://www.postgresql.org/docs/current/functions-json.html#:~:text=jsonb%20%23%2D%20text%5B%5D%20%E2%86%92%20jsonb - AuditRecord.objects.filter(record_type__in=['add_asset', 'update_asset']).update( - details=RawSQL("details #- '{metadata, access}'", []) - ) - - -class Migration(migrations.Migration): - dependencies = [ - ('api', '0010_auditrecord'), - ] - - operations = [ - migrations.RunPython(remove_access_fields), - migrations.RemoveConstraint( - model_name='asset', - name='asset_metadata_no_computed_keys_or_published', - ), - migrations.AddConstraint( - model_name='asset', - constraint=models.CheckConstraint( - check=models.Q( - models.Q( - ('published', False), - models.Q( - ( - 'metadata__has_any_keys', - [ - 'id', - 'access', - 'path', - 'identifier', - 'contentUrl', - 'contentSize', - 'digest', - 'datePublished', - 'publishedBy', - ], - ), - _negated=True, - ), - ), - models.Q( - ('published', True), - ( - 'metadata__has_keys', - [ - 'id', - 'access', - 'path', - 'identifier', - 'contentUrl', - 'contentSize', - 'digest', - 'datePublished', - 'publishedBy', - ], - ), - ), - _connector='OR', - ), - name='asset_metadata_no_computed_keys_or_published', - ), - ), - ] diff --git a/dandiapi/api/migrations/0011_webknossosdataset_webknossosdatalayer_and_more.py b/dandiapi/api/migrations/0011_webknossosdataset_webknossosdatalayer_and_more.py new file mode 100644 index 000000000..d22faeaaa --- /dev/null +++ b/dandiapi/api/migrations/0011_webknossosdataset_webknossosdatalayer_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.1.13 on 2024-09-03 17:25 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0010_usermetadata_webknossos_credential'), + ] + + operations = [ + migrations.CreateModel( + name='WebKnossosDataset', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('webknossos_dataset_name', models.CharField(blank=True, max_length=100, null=True)), + ('webknossos_organization_name', models.CharField(blank=True, max_length=100, null=True)), + ], + ), + migrations.CreateModel( + name='WebKnossosDataLayer', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('asset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='webknossos_datasets', to='api.asset')), + ('webknossos_dataset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='webknossos_datalayers', to='api.webknossosdataset')), + ], + ), + migrations.CreateModel( + name='WebKnossosAnnotation', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('webknossos_annotation_name', models.CharField(blank=True, max_length=100, null=True)), + ('webknossos_organization', models.CharField(blank=True, max_length=100, null=True)), + ('webknossos_annotation_owner_first_name', models.CharField(blank=True, max_length=100, null=True)), + ('webknossos_annotation_owner_last_name', models.CharField(blank=True, max_length=100, null=True)), + ('asset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='webknossos_annotations', to='api.asset')), + ('webknossos_dataset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='webknossos_annotations', to='api.webknossosdataset')), + ], + ), + ] diff --git a/dandiapi/api/migrations/0012_remove_asset_previous.py b/dandiapi/api/migrations/0012_remove_asset_previous.py deleted file mode 100644 index 8dc4e89a8..000000000 --- a/dandiapi/api/migrations/0012_remove_asset_previous.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 4.1.13 on 2024-10-23 15:52 -from __future__ import annotations - -from django.db import migrations - - -class Migration(migrations.Migration): - dependencies = [ - ('api', '0011_asset_access_metadata'), - ] - - operations = [ - migrations.RemoveField( - model_name='asset', - name='previous', - ), - ] diff --git a/dandiapi/api/migrations/0012_webknossosannotation_webknossos_annotation_id.py b/dandiapi/api/migrations/0012_webknossosannotation_webknossos_annotation_id.py new file mode 100644 index 000000000..d63f632b2 --- /dev/null +++ b/dandiapi/api/migrations/0012_webknossosannotation_webknossos_annotation_id.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.13 on 2024-09-17 17:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0011_webknossosdataset_webknossosdatalayer_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='webknossosannotation', + name='webknossos_annotation_id', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/dandiapi/api/migrations/0013_remove_webknossosannotation_asset.py b/dandiapi/api/migrations/0013_remove_webknossosannotation_asset.py new file mode 100644 index 000000000..ca3f50333 --- /dev/null +++ b/dandiapi/api/migrations/0013_remove_webknossosannotation_asset.py @@ -0,0 +1,17 @@ +# Generated by Django 4.1.13 on 2024-09-18 15:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0012_webknossosannotation_webknossos_annotation_id'), + ] + + operations = [ + migrations.RemoveField( + model_name='webknossosannotation', + name='asset', + ), + ] diff --git a/dandiapi/api/migrations/0014_auditrecord_and_more.py b/dandiapi/api/migrations/0014_auditrecord_and_more.py new file mode 100644 index 000000000..717697644 --- /dev/null +++ b/dandiapi/api/migrations/0014_auditrecord_and_more.py @@ -0,0 +1,62 @@ +# Generated by Django 4.2.17 on 2024-12-10 14:30 + +from django.db import migrations, models +from django.db.models.expressions import RawSQL + +def remove_access_fields(apps, _): + Asset = apps.get_model('api.Asset') + AuditRecord = apps.get_model('api.AuditRecord') + + # Use the postgres jsonb '-' operator to delete the 'access' field from metadata + Asset.objects.filter(published=False, metadata__access__isnull=False).update( + metadata=RawSQL("metadata - 'access'", []) + ) + + # Delete access field from existing audit records + # https://www.postgresql.org/docs/current/functions-json.html#:~:text=jsonb%20%23%2D%20text%5B%5D%20%E2%86%92%20jsonb + AuditRecord.objects.filter(record_type__in=['add_asset', 'update_asset']).update( + details=RawSQL("details #- '{metadata, access}'", []) + ) + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0013_remove_webknossosannotation_asset'), + ] + + operations = [ + migrations.CreateModel( + name='AuditRecord', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('dandiset_id', models.IntegerField()), + ('username', models.CharField(max_length=39)), + ('user_email', models.CharField(max_length=254)), + ('user_fullname', models.CharField(max_length=301)), + ('record_type', models.CharField(choices=[('create_dandiset', 'create_dandiset'), ('change_owners', 'change_owners'), ('update_metadata', 'update_metadata'), ('add_asset', 'add_asset'), ('update_asset', 'update_asset'), ('remove_asset', 'remove_asset'), ('create_zarr', 'create_zarr'), ('upload_zarr_chunks', 'upload_zarr_chunks'), ('delete_zarr_chunks', 'delete_zarr_chunks'), ('finalize_zarr', 'finalize_zarr'), ('unembargo_dandiset', 'unembargo_dandiset'), ('publish_dandiset', 'publish_dandiset'), ('delete_dandiset', 'delete_dandiset')], max_length=32)), + ('details', models.JSONField(blank=True)), + ], + ), + migrations.RunPython(remove_access_fields), + migrations.RemoveConstraint( + model_name='asset', + name='asset_metadata_no_computed_keys_or_published', + ), + migrations.RemoveConstraint( + model_name='assetpath', + name='consistent-slash', + ), + migrations.RemoveField( + model_name='asset', + name='previous', + ), + migrations.AddConstraint( + model_name='asset', + constraint=models.CheckConstraint(check=models.Q(models.Q(('published', False), models.Q(('metadata__has_any_keys', ['id', 'access', 'path', 'identifier', 'contentUrl', 'contentSize', 'digest', 'datePublished', 'publishedBy']), _negated=True)), models.Q(('published', True), ('metadata__has_keys', ['id', 'access', 'path', 'identifier', 'contentUrl', 'contentSize', 'digest', 'datePublished', 'publishedBy'])), _connector='OR'), name='asset_metadata_no_computed_keys_or_published'), + ), + migrations.AddConstraint( + model_name='assetpath', + constraint=models.CheckConstraint(check=models.Q(('path__endswith', '/'), ('path__startswith', '/'), _connector='OR', _negated=True), name='consistent-slash'), + ), + ] diff --git a/dandiapi/api/models/__init__.py b/dandiapi/api/models/__init__.py index 082c7bb95..b6867acb2 100644 --- a/dandiapi/api/models/__init__.py +++ b/dandiapi/api/models/__init__.py @@ -8,6 +8,7 @@ from .upload import Upload from .user import UserMetadata from .version import Version +from .webknossos import WebKnossosAnnotation, WebKnossosDataset, WebKnossosDataLayer __all__ = [ 'Asset', @@ -20,4 +21,7 @@ 'Upload', 'UserMetadata', 'Version', + 'WebKnossosAnnotation', + 'WebKnossosDataset', + 'WebKnossosDataLayer' ] diff --git a/dandiapi/api/models/asset.py b/dandiapi/api/models/asset.py index a330c9aba..c455ed1aa 100644 --- a/dandiapi/api/models/asset.py +++ b/dandiapi/api/models/asset.py @@ -1,6 +1,8 @@ from __future__ import annotations import datetime +import logging +import os import re from typing import TYPE_CHECKING from urllib.parse import urlparse, urlunparse @@ -35,6 +37,22 @@ 'publishedBy', ] +logging.basicConfig(level=logging.ERROR) + +def construct_neuroglancer_url(asset_path: str) -> str: + replacement_url = os.getenv('CLOUDFRONT_NEUROGLANCER_URL', None) + if not replacement_url: + return "Neuroglancer not supported" + + parts = asset_path.split('/') + file_type_prefix = parts[3] + cloudfront_s3_location = replacement_url + '/' + '/'.join(parts[3:]) + + if file_type_prefix != 'zarr': + file_type_prefix = 'nifti' + + return f'{file_type_prefix}://{cloudfront_s3_location}' + def validate_asset_path(path: str): if path.startswith('/'): @@ -162,6 +180,13 @@ def is_blob(self): def is_zarr(self): return self.zarr is not None + @property + def is_embargoed(self) -> bool: + if self.blob is not None: + return self.blob.embargoed + + return self.zarr.embargoed # type: ignore # noqa: PGH003 + @property def size(self): if self.is_blob: @@ -187,6 +212,30 @@ def s3_url(self) -> str: return self.blob.s3_url return self.zarr.s3_url + @property + def s3_uri(self) -> str: + if self.s3_url is None: + raise ValueError("s3_url cannot be None") + + s3_url_substring = None + if self.s3_url.startswith("https://"): + s3_url_substring = self.s3_url[len("https://"):] + elif self.s3_url.startswith("http://"): + s3_url_substring = self.s3_url[len("http://"):] + + if s3_url_substring is None: + raise ValueError("s3_url must start with 'https://' or 'http://'") + + bucket_name_end_index = s3_url_substring.find(".s3.amazonaws.com") + bucket_name = s3_url_substring[:bucket_name_end_index] + path = s3_url_substring[bucket_name_end_index + len(".s3.amazonaws.com"):] + + if not path.startswith("/"): + path = "/" + path + + return f"s3://{bucket_name}{path}" + + def is_different_from( self, *, @@ -195,7 +244,7 @@ def is_different_from( metadata: dict, path: str, ) -> bool: - from dandiapi.zarr.models import EmbargoedZarrArchive, ZarrArchive + from dandiapi.zarr.models import ZarrArchive if isinstance(asset_blob, AssetBlob) and self.blob is not None and self.blob != asset_blob: return True @@ -207,16 +256,10 @@ def is_different_from( ): return True - if isinstance(zarr_archive, EmbargoedZarrArchive): - raise NotImplementedError - if self.path != path: return True - if self.metadata != metadata: # noqa: SIM103 - return True - - return False + return self.metadata != metadata @staticmethod def dandi_asset_id(asset_id: str | uuid.UUID): @@ -228,23 +271,31 @@ def full_metadata(self): 'asset-download', kwargs={'asset_id': str(self.asset_id)}, ) + + neuroglancer_url = "Neuroglancer not supported for asset" + try: + neuroglancer_url = construct_neuroglancer_url(self.s3_url) + except Exception: # Catching all exceptions, but logging them + logging.exception("Error constructing neuroglancer URL") + metadata = { **self.metadata, 'id': self.dandi_asset_id(self.asset_id), 'access': [ { 'schemaKey': 'AccessRequirements', - # TODO: When embargoed zarrs land, include that logic here 'status': AccessType.EmbargoedAccess.value - if self.blob and self.blob.embargoed + if self.is_embargoed else AccessType.OpenAccess.value, } ], 'path': self.path, 'identifier': str(self.asset_id), 'contentUrl': [download_url, self.s3_url], + 's3_uri': self.s3_uri, 'contentSize': self.size, 'digest': self.digest, + 'neuroglancerUrl': neuroglancer_url, } schema_version = metadata['schemaVersion'] metadata['@context'] = ( @@ -283,7 +334,4 @@ def total_size(cls): .aggregate(size=models.Sum('size'))['size'] or 0 for cls in (AssetBlob, ZarrArchive) - # adding of Zarrs to embargoed dandisets is not supported - # so no point of adding EmbargoedZarr here since would also result in error - # TODO: add EmbagoedZarr whenever supported ) diff --git a/dandiapi/api/models/user.py b/dandiapi/api/models/user.py index b3c324dae..7a206814f 100644 --- a/dandiapi/api/models/user.py +++ b/dandiapi/api/models/user.py @@ -1,7 +1,10 @@ from __future__ import annotations +import os + from django.contrib.auth.models import User -from django.db import models +from django.db import models, transaction +from django.utils.crypto import get_random_string from django_extensions.db.models import TimeStampedModel @@ -16,3 +19,56 @@ class Status(models.TextChoices): status = models.CharField(choices=Status.choices, default=Status.INCOMPLETE, max_length=10) questionnaire_form = models.JSONField(blank=True, null=True) rejection_reason = models.TextField(blank=True, default='', max_length=1000) + webknossos_credential = models.CharField(max_length=128, blank=True, null=True) # noqa: DJ001 + + def should_register_webknossos_account(self, api_url=None) -> bool: + + return (self.status == self.Status.APPROVED and + not self.webknossos_credential and + api_url) + + def register_webknossos_account(self, webknossos_api_url: str, webknossos_credential: str) -> None: + + webknossos_organization_name = os.getenv('WEBKNOSSOS_ORGANIZATION_NAME', None) + webknossos_organization_display_name = os.getenv('WEBKNOSSOS_ORGANIZATION_DISPLAY_NAME', + None) + + # Offset to celery task to call /register in WebKNOSSOS + from dandiapi.api.tasks import register_external_api_request_task + + register_external_api_request_task.delay( + method='POST', + external_endpoint='https://webknossos.lincbrain.org/api/auth/register', + payload={ + "firstName": self.user.first_name, + "lastName": self.user.last_name, + "email": self.user.email, + "organization": "LINC", + "organizationName": "LINC", + "password": { + "password1": webknossos_credential, + "password2": webknossos_credential + } + } + ) + + def save(self, *args, **kwargs): + + with transaction.atomic(): + + super().save(*args, **kwargs) + + is_new_instance = self.pk is None + if not is_new_instance: + webknossos_api_url = os.getenv('WEBKNOSSOS_API_URL', None) + + if self.should_register_webknossos_account(api_url=webknossos_api_url): + + random_password = get_random_string(length=12) + self.webknossos_credential = random_password + self.register_webknossos_account(webknossos_api_url=webknossos_api_url, webknossos_credential=random_password) + # Slightly recursive call, but will halt with WebKNOSSOS logic + super().save(*args, **kwargs) + + + diff --git a/dandiapi/api/models/webknossos.py b/dandiapi/api/models/webknossos.py new file mode 100644 index 000000000..dd5e75806 --- /dev/null +++ b/dandiapi/api/models/webknossos.py @@ -0,0 +1,81 @@ +from __future__ import annotations +import os + +from uuid import uuid4 + +from django.db import models + +from .asset import Asset + +WEBKNOSSOS_BINARY_DATA_FOLDER = "binaryData" +WEBKNOSSOS_BINARY_DATA_PORT = "8080" +WEBKNOSSOS_DATASOURCE_PROPERTIES_FILE_NAME = "datasource-properties.json" + +class WebKnossosDataset(models.Model): # noqa: DJ008 + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + webknossos_dataset_name = models.CharField(max_length=100, null=True, blank=True) + webknossos_organization_name = models.CharField(max_length=100, null=True, blank=True) + + def get_datasource_properties_url(self) -> str: + webknossos_api_url = os.getenv('WEBKNOSSOS_API_URL', "webknossos.lincbrain.org") + + if webknossos_api_url: + return (f'http://{webknossos_api_url}:{WEBKNOSSOS_BINARY_DATA_PORT}/' + f'{WEBKNOSSOS_BINARY_DATA_FOLDER}/{self.webknossos_organization_name}/' + f'{self.webknossos_dataset_name}/{WEBKNOSSOS_DATASOURCE_PROPERTIES_FILE_NAME}') + raise Exception("WEBKNOSSOS_API_URL is not set") + + def get_asset_s3_uri(self) -> str: + return self.asset.s3_uri + + def get_webknossos_url(self) -> str: + webknossos_api_url = os.getenv('WEBKNOSSOS_API_URL', "webknossos.lincbrain.org") + + if webknossos_api_url: + return (f'https://{webknossos_api_url}/datasets/{self.webknossos_organization_name}/{self.webknossos_dataset_name}') + raise Exception("WEBKNOSSOS_API_URL is not set") + + +class WebKnossosDataLayer(models.Model): + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + webknossos_dataset = models.ForeignKey(WebKnossosDataset, related_name='webknossos_datalayers', on_delete=models.PROTECT) + asset = models.ForeignKey(Asset, related_name='webknossos_datasets', on_delete=models.PROTECT) + + def get_webknossos_url(self) -> str: + webknossos_api_url = os.getenv('WEBKNOSSOS_API_URL', "webknossos.lincbrain.org") + + if webknossos_api_url: + return (f'https://webknossos.lincbrain.org/datasets/LINC' + f'/{self.webknossos_dataset.webknossos_dataset_name}') + # raise Exception("WEBKNOSSOS_API_URL is not set") + +class WebKnossosAnnotation(models.Model): # noqa: DJ008 + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + webknossos_annotation_name = models.CharField(max_length=100, null=True, blank=True) + webknossos_annotation_id = models.CharField(max_length=100, null=True, blank=True) + webknossos_organization = models.CharField(max_length=100, null=True, blank=True) + webknossos_annotation_owner_first_name = models.CharField(max_length=100, null=True, blank=True) + webknossos_annotation_owner_last_name = models.CharField(max_length=100, null=True, blank=True) + + # ForeignKeys + webknossos_dataset = models.ForeignKey(WebKnossosDataset, related_name='webknossos_annotations', on_delete=models.PROTECT) + + def get_asset_s3_uri(self) -> str: + return self.asset.s3_uri + + def get_author_full_name(self) -> str: + return (f'{self.webknossos_annotation_owner_first_name}' + f' {self.webknossos_annotation_owner_last_name}') + + def get_webknossos_url(self) -> str: + webknossos_api_url = os.getenv('WEBKNOSSOS_API_URL', "webknossos.lincbrain.org") + + if webknossos_api_url: + return (f'https://webknossos.lincbrain.org/annotations' + f'/{self.webknossos_annotation_id}') + + def get_webknossos_annotation_name(self) -> str: + if self.webknossos_annotation_name == '': + return "annotation untitled" + + return self.webknossos_annotation_name diff --git a/dandiapi/api/services/asset/__init__.py b/dandiapi/api/services/asset/__init__.py index e1ef3f4d1..834e00520 100644 --- a/dandiapi/api/services/asset/__init__.py +++ b/dandiapi/api/services/asset/__init__.py @@ -168,15 +168,16 @@ def add_asset_to_version( raise ZarrArchiveBelongsToDifferentDandisetError with transaction.atomic(): - # Creating an asset in an OPEN dandiset that points to an embargoed blob results in that - # blob being unembargoed + # Creating an asset in an OPEN dandiset that points to an + # embargoed blob results in that blob being unembargoed. + # NOTE: This only applies to asset blobs, as zarrs cannot belong to + # multiple dandisets at once. if ( asset_blob is not None and asset_blob.embargoed and version.dandiset.embargo_status == Dandiset.EmbargoStatus.OPEN ): - asset_blob.embargoed = False - asset_blob.save() + AssetBlob.objects.filter(blob_id=asset_blob.blob_id).update(embargoed=False) transaction.on_commit( lambda: remove_asset_blob_embargoed_tag_task.delay(blob_id=asset_blob.blob_id) ) diff --git a/dandiapi/api/services/embargo/__init__.py b/dandiapi/api/services/embargo/__init__.py index e2f671998..e7a8841ca 100644 --- a/dandiapi/api/services/embargo/__init__.py +++ b/dandiapi/api/services/embargo/__init__.py @@ -1,67 +1,32 @@ from __future__ import annotations -from concurrent.futures import ThreadPoolExecutor import logging from typing import TYPE_CHECKING -from botocore.config import Config -from django.conf import settings from django.db import transaction -from more_itertools import chunked from dandiapi.api.mail import send_dandiset_unembargoed_message from dandiapi.api.models import AssetBlob, Dandiset, Version from dandiapi.api.services import audit from dandiapi.api.services.asset.exceptions import DandisetOwnerRequiredError +from dandiapi.api.services.embargo.utils import _delete_object_tags, remove_dandiset_embargo_tags from dandiapi.api.services.exceptions import DandiError from dandiapi.api.services.metadata import validate_version_metadata from dandiapi.api.storage import get_boto_client from dandiapi.api.tasks import unembargo_dandiset_task +from dandiapi.zarr.models import ZarrArchive from .exceptions import ( AssetBlobEmbargoedError, - AssetTagRemovalError, DandisetActiveUploadsError, DandisetNotEmbargoedError, ) if TYPE_CHECKING: from django.contrib.auth.models import User - from mypy_boto3_s3 import S3Client logger = logging.getLogger(__name__) -ASSET_BLOB_TAG_REMOVAL_CHUNK_SIZE = 5000 - - -def _delete_asset_blob_tags(client: S3Client, blob: str): - client.delete_object_tagging( - Bucket=settings.DANDI_DANDISETS_BUCKET_NAME, - Key=blob, - ) - - -# NOTE: In testing this took ~2 minutes for 100,000 files -def _remove_dandiset_asset_blob_embargo_tags(dandiset: Dandiset): - client = get_boto_client(config=Config(max_pool_connections=100)) - embargoed_asset_blobs = ( - AssetBlob.objects.filter(embargoed=True, assets__versions__dandiset=dandiset) - .values_list('blob', flat=True) - .iterator(chunk_size=ASSET_BLOB_TAG_REMOVAL_CHUNK_SIZE) - ) - - # Chunk the blobs so we're never storing a list of all embargoed blobs - chunks = chunked(embargoed_asset_blobs, ASSET_BLOB_TAG_REMOVAL_CHUNK_SIZE) - for chunk in chunks: - with ThreadPoolExecutor(max_workers=100) as e: - futures = [ - e.submit(_delete_asset_blob_tags, client=client, blob=blob) for blob in chunk - ] - - # Check if any failed and raise exception if so - failed = [blob for i, blob in enumerate(chunk) if futures[i].exception() is not None] - if failed: - raise AssetTagRemovalError('Some blobs failed to remove tags', blobs=failed) @transaction.atomic() @@ -80,13 +45,17 @@ def unembargo_dandiset(ds: Dandiset, user: User): # Remove tags in S3 logger.info('Removing tags...') - _remove_dandiset_asset_blob_embargo_tags(ds) + remove_dandiset_embargo_tags(ds) - # Update embargoed flag on asset blobs - updated = AssetBlob.objects.filter(embargoed=True, assets__versions__dandiset=ds).update( + # Update embargoed flag on asset blobs and zarrs + updated_blobs = AssetBlob.objects.filter(embargoed=True, assets__versions__dandiset=ds).update( embargoed=False ) - logger.info('Updated %s asset blobs', updated) + updated_zarrs = ZarrArchive.objects.filter( + embargoed=True, assets__versions__dandiset=ds + ).update(embargoed=False) + logger.info('Updated %s asset blobs', updated_blobs) + logger.info('Updated %s zarrs', updated_zarrs) # Set status to OPEN Dandiset.objects.filter(pk=ds.pk).update(embargo_status=Dandiset.EmbargoStatus.OPEN) @@ -118,7 +87,7 @@ def remove_asset_blob_embargoed_tag(asset_blob: AssetBlob) -> None: if asset_blob.embargoed: raise AssetBlobEmbargoedError - _delete_asset_blob_tags(client=get_boto_client(), blob=asset_blob.blob.name) + _delete_object_tags(client=get_boto_client(), blob=asset_blob.blob.name) def kickoff_dandiset_unembargo(*, user: User, dandiset: Dandiset): diff --git a/dandiapi/api/services/embargo/utils.py b/dandiapi/api/services/embargo/utils.py new file mode 100644 index 000000000..1b7c405ff --- /dev/null +++ b/dandiapi/api/services/embargo/utils.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import logging +from typing import TYPE_CHECKING + +from botocore.config import Config +from django.conf import settings +from django.db.models import Q +from more_itertools import chunked + +from dandiapi.api.models.asset import Asset +from dandiapi.api.storage import get_boto_client +from dandiapi.zarr.models import zarr_s3_path + +from .exceptions import AssetTagRemovalError + +if TYPE_CHECKING: + from mypy_boto3_s3 import S3Client + + from dandiapi.api.models.dandiset import Dandiset + + +logger = logging.getLogger(__name__) +TAG_REMOVAL_CHUNK_SIZE = 5000 + + +def retry(times: int, exceptions: tuple[type[Exception]]): + """ + Retry Decorator. + + Retries the wrapped function/method `times` times if the exceptions listed + in ``exceptions`` are thrown + + :param times: The number of times to repeat the wrapped function/method + :param exceptions: Lists of exceptions that trigger a retry attempt + """ + + def decorator(func): + def newfn(*args, **kwargs): + attempt = 0 + while attempt < times: + try: + return func(*args, **kwargs) + except exceptions: + attempt += 1 + return func(*args, **kwargs) + + return newfn + + return decorator + + +@retry(times=3, exceptions=(Exception,)) +def _delete_object_tags(client: S3Client, blob: str): + client.delete_object_tagging( + Bucket=settings.DANDI_DANDISETS_BUCKET_NAME, + Key=blob, + ) + + +@retry(times=3, exceptions=(Exception,)) +def _delete_zarr_object_tags(client: S3Client, zarr: str): + paginator = client.get_paginator('list_objects_v2') + pages = paginator.paginate( + Bucket=settings.DANDI_DANDISETS_BUCKET_NAME, Prefix=zarr_s3_path(zarr_id=zarr) + ) + + with ThreadPoolExecutor(max_workers=100) as e: + for page in pages: + keys = [obj['Key'] for obj in page.get('Contents', [])] + futures = [e.submit(_delete_object_tags, client=client, blob=key) for key in keys] + + # Check if any failed and raise exception if so + failed = [key for i, key in enumerate(keys) if futures[i].exception() is not None] + if failed: + raise AssetTagRemovalError('Some zarr files failed to remove tags', blobs=failed) + + +def remove_dandiset_embargo_tags(dandiset: Dandiset): + client = get_boto_client(config=Config(max_pool_connections=100)) + embargoed_assets = ( + Asset.objects.filter(versions__dandiset=dandiset) + .filter(Q(blob__embargoed=True) | Q(zarr__embargoed=True)) + .values_list('blob__blob', 'zarr__zarr_id') + .iterator(chunk_size=TAG_REMOVAL_CHUNK_SIZE) + ) + + # Chunk the blobs so we're never storing a list of all embargoed blobs + chunks = chunked(embargoed_assets, TAG_REMOVAL_CHUNK_SIZE) + for chunk in chunks: + futures = [] + with ThreadPoolExecutor(max_workers=100) as e: + for blob, zarr in chunk: + if blob is not None: + futures.append(e.submit(_delete_object_tags, client=client, blob=blob)) + if zarr is not None: + futures.append(e.submit(_delete_zarr_object_tags, client=client, zarr=zarr)) + + # Check if any failed and raise exception if so + failed = [blob for i, blob in enumerate(chunk) if futures[i].exception() is not None] + if failed: + raise AssetTagRemovalError('Some assets failed to remove tags', blobs=failed) diff --git a/dandiapi/api/services/metadata/__init__.py b/dandiapi/api/services/metadata/__init__.py index 0656916a1..097b2d649 100644 --- a/dandiapi/api/services/metadata/__init__.py +++ b/dandiapi/api/services/metadata/__init__.py @@ -7,6 +7,7 @@ from dandischema.metadata import aggregate_assets_summary, validate from django.conf import settings from django.db import transaction +from django.db.models.query_utils import Q from django.utils import timezone from dandiapi.api.models import Asset, Version @@ -124,7 +125,11 @@ def _build_validatable_version_metadata(version: Version) -> dict: metadata_for_validation['doi'] = '10.80507/dandi.123456/0.123456.1234' metadata_for_validation['assetsSummary'] = { 'schemaKey': 'AssetsSummary', - 'numberOfBytes': 1 if version.assets.filter(blob__size__gt=0).exists() else 0, + 'numberOfBytes': 1 + if version.assets.filter( + Q(blob__size__gt=0) | Q(zarr__size__gt=0) + ).exists() + else 0, 'numberOfFiles': 1 if version.assets.exists() else 0, } return metadata_for_validation @@ -172,7 +177,7 @@ def _get_version_validation_result( # If the version has since been modified, return early if current_version.status != Version.Status.PENDING: logger.info( - 'Skipping validation for version with a status of %s', current_version.status + 'Skipping validation for version %s due to concurrent modification', version_id ) return diff --git a/dandiapi/api/services/publish/__init__.py b/dandiapi/api/services/publish/__init__.py index 882fa0c7c..f3a2ecb5b 100644 --- a/dandiapi/api/services/publish/__init__.py +++ b/dandiapi/api/services/publish/__init__.py @@ -55,7 +55,7 @@ def _lock_dandiset_for_publishing(*, user: User, dandiset: Dandiset) -> None: # if dandiset.embargo_status != Dandiset.EmbargoStatus.OPEN: raise NotAllowedError('Operation only allowed on OPEN dandisets', 400) - if dandiset.zarr_archives.exists() or dandiset.embargoed_zarr_archives.exists(): + if dandiset.zarr_archives.exists(): raise NotAllowedError('Cannot publish dandisets which contain zarrs', 400) with transaction.atomic(): diff --git a/dandiapi/api/services/publish/exceptions.py b/dandiapi/api/services/publish/exceptions.py index f2310d9c1..4f815ff03 100644 --- a/dandiapi/api/services/publish/exceptions.py +++ b/dandiapi/api/services/publish/exceptions.py @@ -12,22 +12,22 @@ class DandisetAlreadyPublishedError(DandiError): class DandisetAlreadyPublishingError(DandiError): http_status_code = status.HTTP_423_LOCKED - message = 'Dandiset is currently being published' + message = 'Dataset is currently being published' class DandisetBeingValidatedError(DandiError): http_status_code = status.HTTP_409_CONFLICT - message = 'Dandiset is currently being validated' + message = 'Dataset is currently being validated' class DandisetInvalidMetadataError(DandiError): http_status_code = status.HTTP_400_BAD_REQUEST - message = 'Dandiset metadata or asset metadata is not valid' + message = 'Dataset metadata or asset metadata is not valid' class DandisetValidationPendingError(DandiError): http_status_code = status.HTTP_409_CONFLICT - message = 'Metadata validation is pending for this dandiset, please try again later.' + message = 'Metadata validation is pending for this dataset, please try again later.' class DandisetNotLockedError(DandiError): diff --git a/dandiapi/api/tasks/__init__.py b/dandiapi/api/tasks/__init__.py index 2b4496c74..404d57523 100644 --- a/dandiapi/api/tasks/__init__.py +++ b/dandiapi/api/tasks/__init__.py @@ -5,6 +5,7 @@ from celery import shared_task from celery.exceptions import SoftTimeLimitExceeded from celery.utils.log import get_task_logger +import requests from django.contrib.auth.models import User from dandiapi.api.doi import delete_doi @@ -91,6 +92,35 @@ def publish_dandiset_task(dandiset_id: int, user_id: int): _publish_dandiset(dandiset_id=dandiset_id, user_id=user_id) +@shared_task +def register_external_api_request_task(method: str, external_endpoint: str, payload: dict = None, + query_params: dict = None): + headers = { + 'Content-Type': 'application/json', + } + try: + if method.upper() == 'POST': + response = requests.post(external_endpoint, json=payload, headers=headers, timeout=10) + response.raise_for_status() + logger.info(f"POST to {external_endpoint} successful with payload: {payload}") + logger.info(f"Response: {response.status_code}, {response.text}") + elif method.upper() == 'GET': + response = requests.get(external_endpoint, params=query_params, headers=headers, + timeout=10) + logger.info(f"GET to {external_endpoint} successful") + logger.info(f"Response: {response.status_code}, {response.headers}") + return {'status_code': response.status_code, 'headers': response.headers} + else: + logger.error("Unsupported HTTP method: %s", method) + return + except requests.exceptions.HTTPError as http_err: + logger.error(f"HTTP error occurred: {http_err}") + logger.error(f"Response content: {response.text}") # Log response in case of error + except requests.exceptions.RequestException as req_err: + logger.error(f"Request exception occurred: {req_err}") + except Exception as err: + logger.error(f"An unexpected error occurred: {err}") + @shared_task(soft_time_limit=1200) def unembargo_dandiset_task(dandiset_id: int, user_id: int): from dandiapi.api.services.embargo import unembargo_dandiset diff --git a/dandiapi/api/tasks/scheduled.py b/dandiapi/api/tasks/scheduled.py index 2b4a100e9..ff9e03325 100644 --- a/dandiapi/api/tasks/scheduled.py +++ b/dandiapi/api/tasks/scheduled.py @@ -20,9 +20,10 @@ from django.db.models.query_utils import Q from dandiapi.analytics.tasks import collect_s3_log_records_task -from dandiapi.api.mail import send_pending_users_message +from dandiapi.api.mail import send_dandisets_to_unembargo_message, send_pending_users_message from dandiapi.api.models import UserMetadata, Version from dandiapi.api.models.asset import Asset +from dandiapi.api.models.dandiset import Dandiset from dandiapi.api.services.metadata import version_aggregate_assets_summary from dandiapi.api.services.metadata.exceptions import VersionMetadataConcurrentlyModifiedError from dandiapi.api.tasks import ( @@ -30,6 +31,7 @@ validate_version_metadata_task, write_manifest_files, ) +from dandiapi.api.views.auth import populate_webknossos_datasets_and_annotations from dandiapi.zarr.models import ZarrArchiveStatus if TYPE_CHECKING: @@ -111,6 +113,14 @@ def send_pending_users_email() -> None: send_pending_users_message(pending_users) +@shared_task(soft_time_limit=20) +def send_dandisets_to_unembargo_email() -> None: + """Send an email to admins listing dandisets that have requested unembargo.""" + dandisets = Dandiset.objects.filter(embargo_status=Dandiset.EmbargoStatus.UNEMBARGOING) + if dandisets.exists(): + send_dandisets_to_unembargo_message(dandisets) + + @shared_task(soft_time_limit=60) def refresh_materialized_view_search() -> None: """ @@ -123,6 +133,10 @@ def refresh_materialized_view_search() -> None: cursor.execute('REFRESH MATERIALIZED VIEW CONCURRENTLY asset_search;') +@shared_task(soft_time_limit=100) +def populate_webknossos_datasets_and_annotations_task() -> None: + populate_webknossos_datasets_and_annotations({}, 'webknossos') + def register_scheduled_tasks(sender: Celery, **kwargs): """Register tasks with a celery beat schedule.""" # Check for any draft versions that need validation every minute @@ -139,8 +153,16 @@ def register_scheduled_tasks(sender: Celery, **kwargs): # Send daily email to admins containing a list of users awaiting approval sender.add_periodic_task(crontab(hour=0, minute=0), send_pending_users_email.s()) + # Send daily email to admins containing a list of dandisets to unembargo + sender.add_periodic_task(crontab(hour=0, minute=0), send_dandisets_to_unembargo_email.s()) + # Refresh the materialized view used by asset search every 10 mins. sender.add_periodic_task(timedelta(minutes=10), refresh_materialized_view_search.s()) # Process new S3 logs every hour sender.add_periodic_task(timedelta(hours=1), collect_s3_log_records_task.s()) + + sender.add_periodic_task( + crontab(hour=12, minute=0), + populate_webknossos_datasets_and_annotations_task.s() + ) diff --git a/dandiapi/api/templates/api/account/base.html b/dandiapi/api/templates/api/account/base.html index ecfdfda91..08f7c684c 100644 --- a/dandiapi/api/templates/api/account/base.html +++ b/dandiapi/api/templates/api/account/base.html @@ -9,7 +9,7 @@ - DANDI Archive + LINC Data Platform diff --git a/web/src/views/DandisetLandingView/ShareDialog.vue b/web/src/views/DandisetLandingView/ShareDialog.vue index e01c568fe..14bb3015a 100644 --- a/web/src/views/DandisetLandingView/ShareDialog.vue +++ b/web/src/views/DandisetLandingView/ShareDialog.vue @@ -39,26 +39,26 @@ - Dandiset link: + Dataset link: - - - - DOI link: - - + + + + + + + + + + + + diff --git a/web/src/views/FileBrowserView/FileBrowser.vue b/web/src/views/FileBrowserView/FileBrowser.vue index a18072688..138b20765 100644 --- a/web/src/views/FileBrowserView/FileBrowser.vue +++ b/web/src/views/FileBrowserView/FileBrowser.vue @@ -161,7 +161,6 @@ Open asset in browser (you can also click on the item itself) - @@ -50,7 +48,7 @@ import { watchEffect } from 'vue'; import { useRoute, useRouter } from 'vue-router/composables'; import StatsBar from '@/views/HomeView/StatsBar.vue'; import DandisetSearchField from '@/components/DandisetSearchField.vue'; -import logo from '@/assets/logo.svg'; +import logo from '@/assets/linc-logo.svg'; /** * Redirect old hash URLS to the correct one. This is only done on diff --git a/web/src/views/HomeView/StatsBar.vue b/web/src/views/HomeView/StatsBar.vue index 9242301a5..92f794368 100644 --- a/web/src/views/HomeView/StatsBar.vue +++ b/web/src/views/HomeView/StatsBar.vue @@ -36,9 +36,9 @@ const size = ref(0); const stats = computed(() => [ { - name: 'dandisets', + name: 'datasets', value: dandisets.value, - description: 'A DANDI dataset including files and dataset-level metadata', + description: 'A dataset including files and dataset-level metadata', href: '/dandiset', }, { name: 'users', value: users.value }, diff --git a/web/src/views/MyDandisetsView/MyDandisetsView.vue b/web/src/views/MyDandisetsView/MyDandisetsView.vue index 10efb339c..04c9d0772 100644 --- a/web/src/views/MyDandisetsView/MyDandisetsView.vue +++ b/web/src/views/MyDandisetsView/MyDandisetsView.vue @@ -1,11 +1,11 @@