Skip to content

Commit

Permalink
[red-knot] Property test improvements (#15358)
Browse files Browse the repository at this point in the history
## Summary

- Add a workflow to run property tests on a daily basis (based on
`daily_fuzz.yaml`)
- Mark `assignable_to_is_reflexive` as flaky (related to #14899)
- Add new (failing) `intersection_assignable_to_both` test (also related
to #14899)

## Test Plan

Ran:

```bash
export QUICKCHECK_TESTS=100000
while cargo test --release -p red_knot_python_semantic -- \
  --ignored types::property_tests::stable; do :; done
```

Observed successful property_tests CI run
  • Loading branch information
sharkdp authored Jan 8, 2025
1 parent beb8e2d commit 4fd82d5
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 6 deletions.
71 changes: 71 additions & 0 deletions .github/workflows/daily_property_tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
name: Daily property test run

on:
workflow_dispatch:
schedule:
- cron: "0 12 * * *"
pull_request:
paths:
- ".github/workflows/daily_property_tests.yaml"

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

env:
CARGO_INCREMENTAL: 0
CARGO_NET_RETRY: 10
CARGO_TERM_COLOR: always
RUSTUP_MAX_RETRIES: 10
FORCE_COLOR: 1

jobs:
property_tests:
name: Property tests
runs-on: ubuntu-latest
timeout-minutes: 20
# Don't run the cron job on forks:
if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- name: "Install Rust toolchain"
run: rustup show
- name: "Install mold"
uses: rui314/setup-mold@v1
- uses: Swatinem/rust-cache@v2
- name: Build Red Knot
# A release build takes longer (2 min vs 1 min), but the property tests run much faster in release
# mode (1.5 min vs 14 min), so the overall time is shorter with a release build.
run: cargo build --locked --release --package red_knot_python_semantic --tests
- name: Run property tests
shell: bash
run: |
export QUICKCHECK_TESTS=100000
for _ in {1..5}; do
cargo test --locked --release --package red_knot_python_semantic -- --ignored types::property_tests::stable
done
create-issue-on-failure:
name: Create an issue if the daily property test run surfaced any bugs
runs-on: ubuntu-latest
needs: property_tests
if: ${{ github.repository == 'astral-sh/ruff' && always() && github.event_name == 'schedule' && needs.property_tests.result == 'failure' }}
permissions:
issues: write
steps:
- uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.create({
owner: "astral-sh",
repo: "ruff",
title: `Daily property test run failed on ${new Date().toDateString()}`,
body: "Runs listed here: https://github.com/astral-sh/ruff/actions/workflows/daily_property_tests.yaml",
labels: ["bug", "red_knot", "testing"],
})
32 changes: 26 additions & 6 deletions crates/red_knot_python_semantic/src/types/property_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -258,12 +258,6 @@ mod stable {
forall types s, t. s.is_subtype_of(db, t) => !s.is_disjoint_from(db, t) || s.is_never()
);

// `T` can be assigned to itself.
type_property_test!(
assignable_to_is_reflexive, db,
forall types t. t.is_assignable_to(db, t)
);

// `S <: T` implies that `S` can be assigned to `T`.
type_property_test!(
subtype_of_implies_assignable_to, db,
Expand Down Expand Up @@ -321,6 +315,32 @@ mod stable {
/// tests to the `stable` section. In the meantime, it can still be useful to run these
/// tests (using [`types::property_tests::flaky`]), to see if there are any new obvious bugs.
mod flaky {
use crate::{
db::tests::TestDb,
types::{IntersectionBuilder, Type},
};

// Currently fails due to https://github.com/astral-sh/ruff/issues/14899
// `T` can be assigned to itself.
type_property_test!(
assignable_to_is_reflexive, db,
forall types t. t.is_assignable_to(db, t)
);

// Currently fails due to https://github.com/astral-sh/ruff/issues/14899
// An intersection of two types should be assignable to both of them
fn intersection<'db>(db: &'db TestDb, s: Type<'db>, t: Type<'db>) -> Type<'db> {
IntersectionBuilder::new(db)
.add_positive(s)
.add_positive(t)
.build()
}

type_property_test!(
intersection_assignable_to_both, db,
forall types s, t. intersection(db, s, t).is_assignable_to(db, s) && intersection(db, s, t).is_assignable_to(db, t)
);

// `S <: T` and `T <: S` implies that `S` is equivalent to `T`.
type_property_test!(
subtype_of_is_antisymmetric, db,
Expand Down

0 comments on commit 4fd82d5

Please sign in to comment.