Merge branch 'master' into jferg/ignore-feedbacks-in-search

.github/CODEOWNERS

@@ -225,10 +225,15 @@ yarn.lock                                                @getsentry/owners-js-de
 
 /static/app/views/performance/                                            @getsentry/performance
 /static/app/utils/performance/                                            @getsentry/performance
-/static/app/components/events/eventStatisticalDetector/                   @getsentry/performance
 /static/app/components/events/interfaces/spans/                           @getsentry/performance
 /static/app/components/performance/                                       @getsentry/performance
 /static/app/components/quickTrace/                                        @getsentry/performance
+
+static/app/components/events/eventStatisticalDetector/                    @getsentry/performance @getsentry/profiling
+/src/sentry/statistical_detectors                                         @getsentry/performance @getsentry/profiling
+/src/sentry/tasks/statistical_detectors.py                                @getsentry/performance @getsentry/profiling
+/tests/sentry/statistical_detectors                                       @getsentry/performance @getsentry/profiling
+/tests/sentry/tasks/test_statistical_detectors.py                         @getsentry/performance @getsentry/profiling
 ## Endof Performance
 
 

.github/workflows/codecov_ats.yml

@@ -44,10 +44,6 @@ jobs:
         if: needs.files-changed.outputs.backend == 'true'
         needs: files-changed
         runs-on: ubuntu-latest
-        # Map a step output to a job output
-        outputs:
-          ATS_TESTS_TO_RUN: ${{ steps.codecov_automated_test_selection.outputs.ATS_TESTS_TO_RUN }}
-          ATS_TESTS_TO_SKIP: ${{ steps.codecov_automated_test_selection.outputs.ATS_TESTS_TO_SKIP }}
         steps:
         - uses: actions/checkout@v3
           with:
@@ -73,7 +69,7 @@ jobs:
             pg-version: 14
         - name: Download Codecov CLI
           run: |
-            pip install --extra-index-url https://pypi.org/simple --no-cache-dir codecov-cli>=0.4.0
+            pip install --extra-index-url https://pypi.org/simple --no-cache-dir codecov-cli>=0.4.1
         # Creates the commit and report objects in codecov
         - name: Codecov startup
           run: |
@@ -98,25 +94,38 @@ jobs:
         - name: Codecov Automated Test Selection
           id: codecov_automated_test_selection
           run: |
+            # Directory for the artifacts from this step
+            mkdir .artifacts/codecov_ats
+            # This is the base for the git diff BASE..HEAD
             BASE_COMMIT=$(git merge-base ${{ github.sha }}^ origin/master)
-            echo $BASE_COMMIT
+            # Get list of tests to run from Codecov
             output=$(codecovcli --codecov-yml-path=codecov.yml label-analysis --dry-run --token=${CODECOV_STATIC_TOKEN} --base-sha=${BASE_COMMIT}) || true
+            # Post processing and validation
             if [ -n "${output}" ];
             then
 
-              echo ATS_TESTS_TO_RUN=$(jq <<< $output '.runner_options + .ats_tests_to_run | @json | @sh' --raw-output) >> "$GITHUB_OUTPUT"
-              echo ATS_TESTS_TO_SKIP=$(jq <<< $output '.runner_options + .ats_tests_to_skip | @json | @sh' --raw-output) >> "$GITHUB_OUTPUT"
+              jq <<< $output '.runner_options + .ats_tests_to_run | @json' --raw-output > .artifacts/codecov_ats/tests_to_run.json
+              jq <<< $output '.runner_options + .ats_tests_to_skip | @json' --raw-output > .artifacts/codecov_ats/tests_to_skip.json
 
               testcount() { jq <<< $output ".$1 | length"; }
               run_count=$(testcount ats_tests_to_run)
               skip_count=$(testcount ats_tests_to_skip)
-              tee <<< "Selected $run_count / $(($run_count + $skip_count)) tests to run" "$GITHUB_STEP_SUMMARY"
+              # Parse any potential errors that made ATS fallback to running all tests
+              # And surface them
+              ats_fallback_reason=$(jq <<< "$output" '.ats_fallback_reason')
+              if [ "$ats_fallback_reason" == "null" ]; then
+                ats_success=true
+              else
+                ats_success=false
+              fi
+              tee <<< \
+                "{\"ats_success\": $ats_success, \"error\": $ats_fallback_reason, \"tests_to_run\": $run_count, \"tests_analyzed\": $((run_count+skip_count))}" \
+                "$GITHUB_STEP_SUMMARY" \
+                ".artifacts/codecov_ats/result.json"
             else
-              tee <<< "ATS failed. Can't get list of tests to run. Fallback to all tests" "$GITHUB_STEP_SUMMARY"
               # We need not forget to add the search options in the fallback command, otherwise pytest might run more tests than expected
               # These search options match what's defined in codecov.yml:105
-              echo 'ATS_TESTS_TO_RUN<<EOF' >> $GITHUB_OUTPUT
-              jq -c @json <<< '[
+              jq '@json' --raw-output <<< '[
                 "--cov-context=test",
                 "tests/sentry",
                 "tests/integration",
@@ -126,44 +135,53 @@ jobs:
                 "--ignore=tests/sentry/search/events",
                 "--ignore=tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py",
                 "--ignore=tests/sentry/region_to_control/test_region_to_control_kafka.py"
-              ]' >> $GITHUB_OUTPUT
-              echo 'EOF' >> $GITHUB_OUTPUT
-              echo ATS_TESTS_TO_SKIP="'[]'" >> "$GITHUB_OUTPUT"
-              echo "::error ATS failed"
-              exit 1
+              ]' > .artifacts/codecov_ats/tests_to_skip.json
+              echo '[]' > .artifacts/codecov_ats/tests_to_run.json
+              # If we reached this point it means that ATS failed with some error
+              tee <<< '{"ats_success": false, "error": "exception_raised"}' "$GITHUB_STEP_SUMMARY" ".artifacts/codecov_ats/result.json"
             fi
           env:
             CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
             CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
+        - uses: actions/upload-artifact@v3
+          with:
+            name: codecov_ats
+            path: .artifacts/codecov_ats
+            if-no-files-found: error
     # The actual running of tests would come here, after the labels are available
     # Something like pytest <options> $ATS_TESTS_TO_RUN
     debug:
       runs-on: ubuntu-latest
-      needs: coverage-ats
-      if: ${{ always() }}
+      needs:
+        - coverage-ats
+        - files-changed
+      # Avoids running this job if it's a frontend change
+      # It would fail if the coverage-ats step didn't run
+      if: needs.files-changed.outputs.backend == 'true'
       steps:
+        - uses: actions/download-artifact@v3
+          with:
+            name: codecov_ats
+            path: .artifacts
         - name: Debug ATS_TESTS_TO_RUN
           run: |
-            : ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }}
-            length_of_tests=$(jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }} 'length')
+            length_of_tests=$(cat .artifacts/tests_to_run.json | jq 'length')
             # The 1st value doesn't count, it's '--cov-context=test' (hence -gt 1)
             if [ $length_of_tests -gt 1 ]; then
               echo "Running $length_of_tests tests"
               # --raw-output0 doesn't work.
-              jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }} 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo
+              cat .artifacts/tests_to_run.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
             else
               echo "No tests to run"
             fi
         - name: Debug ATS_TESTS_TO_SKIP
           run: |
-            : ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }}
-            ATS_TESTS_TO_SKIP='${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }}'
-            length_of_tests=$(jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }} 'length')
+            length_of_tests=$(cat .artifacts/tests_to_skip.json | jq 'length')
             # The 1st value doesn't count, it's '--cov-context=test'
             if [ $length_of_tests -gt 1 ]; then
               echo "Running $length_of_tests tests"
               # --raw-output0 doesn't work.
-              jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }} 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo
+              cat .artifacts/tests_to_skip.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
             else
               echo "No tests to run"
             fi

.github/workflows/codecov_carryforward_reports.yml

@@ -0,0 +1,45 @@
+name: Carry forward codecov reports
+# This workflow carries forward coverage reports for commits in master
+# The coverage reports are generated by .github/workflows/codecov_per_test_coverage.yml
+# By carrying forward the reports and uploading the static analysis information
+# We can use the commits in master as the BASE for Automated Test Selection
+# see .github/workflows/codecov_ats.yml
+
+on:
+  push:
+    branches: [master]
+
+jobs:
+  carryforward-reports-and-upload-static-analysis:
+      runs-on: ubuntu-latest
+      steps:
+        - uses: actions/checkout@v3
+        - name: Set up Python 3.10.10
+          uses: actions/setup-python@v4
+          with:
+            python-version: "3.10.10"
+        - name: Download Codecov CLI
+          run: |
+            pip install --extra-index-url https://pypi.org/simple --no-cache-dir pytest codecov-cli==0.4.0
+        # Creates the commit and report objects in codecov
+        # This carries forward previouly uploaded coverage reports to the new commit
+        - name: Codecov startup
+          run: |
+            codecovcli create-commit
+            codecovcli create-report
+          env:
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        # Sends static analysis information to codecov
+        # This is used as an input in Codecov Automated Test Selection.
+        # It's necessary so we can use this commit as the BASE for comparison
+        - name: Static Analysis
+          run: |
+            codecovcli static-analysis --token=${CODECOV_STATIC_TOKEN} \
+            --folders-to-exclude .artifacts \
+            --folders-to-exclude .github \
+            --folders-to-exclude .venv \
+            --folders-to-exclude static \
+            --folders-to-exclude bin
+          env:
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+            CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}

.github/workflows/codecov_per_test_coverage.yml

@@ -0,0 +1,75 @@
+name: Codecov - per test coverage
+# This workflow generates pytest coverage with the flag --cov-context=test
+# This coverage is used as input for Codecov Automated Test Selection (see .github/workflows/codecov_ats.yml)
+# However there's a performance toll in running tests with this flag.
+# So we will not be running the test suite on every commit
+
+on: [workflow_dispatch, workflow_call]
+
+jobs:
+  # Same as 'backend' in .github/workflows/backed.yml
+  # Except for run_backend_tests step (which includes the extra --cov-context=test flag)
+  # And the coverage generation and handling
+  backend-test-with-cov-context:
+      if: github.ref == 'refs/heads/master'
+      name: backend test
+      runs-on: ubuntu-20.04
+      timeout-minutes: 120
+      strategy:
+        # This helps not having to run multiple jobs because one fails, thus, reducing resource usage
+        # and reducing the risk that one of many runs would turn red again (read: intermittent tests)
+        fail-fast: false
+        matrix:
+          # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
+          instance: [0, 1, 2, 3, 4, 5, 6]
+          pg-version: ['14']
+
+      env:
+        # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`.
+        # If this increases, make sure to also increase `flags.backend.after_n_builds` in `codecov.yml`.
+        MATRIX_INSTANCE_TOTAL: 7
+
+      steps:
+        - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+          with:
+            # Avoid codecov error message related to SHA resolution:
+            # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891
+            fetch-depth: '2'
+
+        - name: Setup sentry env
+          uses: ./.github/actions/setup-sentry
+          id: setup
+          with:
+            kafka: true
+            snuba: true
+            symbolicator: true
+            # Right now, we run so few bigtable related tests that the
+            # overhead of running bigtable in all backend tests
+            # is way smaller than the time it would take to run in its own job.
+            bigtable: true
+            pg-version: ${{ matrix.pg-version }}
+
+        - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) with --cov-context=test
+          id: run_backend_tests
+          run: |
+            make test-python-ci COV_ARGS=--cov-context=test
+
+        # Separate from the testing step above so that we always create the report
+        # Even if some tests fail
+        - name: Create coverage report in JSON format
+          if: ${{ always() }}
+          run: |
+            coverage json --show-contexts -o .artifacts/python.coverage.json
+
+        # Upload coverage data even if running the tests step fails since
+        # it reduces large coverage fluctuations
+        - name: Upload coverage - special case to test Codecov ATS
+          if: ${{ always() }}
+          uses: codecov/codecov-action@v4-beta
+          env:
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          with:
+            files: .artifacts/python.coverage.codecov.json
+            flags: smart-tests
+            plugins: compress-pycoverage
+          continue-on-error: true

.github/workflows/shuffle-tests.yml

@@ -3,6 +3,11 @@ name: shuffle-tests
 on:
   # Allow manually running
   workflow_dispatch:
+    inputs:
+      per-test-coverage:
+        description: Whether to get per-test coverage (uses ./github/workflows/codecov_per_test_coverage.yml)
+        required: true
+        default: 'true'
   # Run once a week on sunday
   schedule:
     - cron: '0 1 * * 0'
@@ -15,6 +20,10 @@ env:
   SENTRY_SHUFFLE_TESTS: true
 
 jobs:
+  per-test-coverage:
+    if: ${{ inputs.per-test-coverage == 'true' || github.event_name == 'schedule' }}
+    uses: ./.github/workflows/codecov_per_test_coverage.yml
+    secrets: inherit
   backend-test:
     name: run backend tests
     runs-on: ubuntu-20.04

.vscode/settings.json

@@ -70,20 +70,9 @@
     "editor.formatOnSave": false
   },
 
-  "python.linting.pylintEnabled": false,
-  "python.linting.flake8Enabled": true,
-  // test discovery is sluggish and the UI around running
-  // tests is often in your way and misclicked
+  "editor.tabSize": 4,
   "python.testing.pytestEnabled": true,
   "python.testing.unittestEnabled": false,
-  "python.testing.nosetestsEnabled": false,
-  "editor.tabSize": 4,
-  "restructuredtext.confPath": "",
-  "python.linting.enabled": true,
-  "python.linting.flake8Path": "${workspaceFolder}/.venv/bin/flake8",
-  "python.linting.pycodestylePath": "${workspaceFolder}/.venv/bin/pep8",
   "python.testing.pytestPath": "${workspaceFolder}/.venv/bin/pytest",
-  "python.formatting.blackPath": "${workspaceFolder}/.venv/bin/black",
-  "python.formatting.provider": "none",
   "python.testing.pytestArgs": ["tests"]
 }

Makefile

@@ -125,6 +125,13 @@ test-js-ci: node-version-check
 	@yarn run test-ci
 	@echo ""
 
+# COV_ARGS controls extra args passed to pytest to generate covereage
+# It's used in test-python-ci. Typically generated an XML coverage file
+# Except in .github/workflows/codecov_per_test_coverage.yml
+# When it's dynamically changed to include --cov-context=test flag
+# See that workflow for more info
+COV_ARGS = --cov-report="xml:.artifacts/python.coverage.xml"
+
 test-python-ci: create-db
 	@echo "--> Running CI Python tests"
 	pytest \
@@ -133,7 +140,7 @@ test-python-ci: create-db
 		--ignore tests/apidocs \
 		--ignore tests/js \
 		--ignore tests/tools \
-		--cov . --cov-report="xml:.artifacts/python.coverage.xml"
+		--cov . $(COV_ARGS)
 	@echo ""
 
 # it's not possible to change settings.DATABASE after django startup, so

babel.config.ts

@@ -44,12 +44,14 @@ const config: TransformOptions = {
           },
         ],
         ['babel-plugin-add-react-displayname'],
+        ['@fullstory/babel-plugin-annotate-react'],
       ],
     },
     development: {
       plugins: [
         '@emotion/babel-plugin',
         '@babel/plugin-transform-react-jsx-source',
+        ['@fullstory/babel-plugin-annotate-react'],
         ...(process.env.SENTRY_UI_HOT_RELOAD ? ['react-refresh/babel'] : []),
       ],
     },