Add support for expected failures (#40)

Makefile

@@ -18,7 +18,7 @@ help: ## Describe useful make targets
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "%-15s %s\n", $$1, $$2}'
 
 .PHONY: all
-all: test lint ## Run all tests and lint (default)
+all: generate lint test conformance ## Run all tests and lint (default)
 
 .PHONY: clean
 clean: ## Delete intermediate build artifacts

docs/conformance.md

@@ -21,7 +21,7 @@ make conformance
 You can customize the behavior of the tests using the following flags:
 
 | Flag                   | Description                                          | Default Value |
-| ---------------------- | ---------------------------------------------------- | ------------- |
+|------------------------|------------------------------------------------------|---------------|
 | `--suite <regex>`      | Filter suites using the provided regular expression. | None          |
 | `--case <regex>`       | Filter cases using the provided regular expression.  | None          |
 | `--timeout <duration>` | Set a per-suite timeout.                             | 5 seconds     |
@@ -30,6 +30,7 @@ You can customize the behavior of the tests using the following flags:
 | `--json`               | Return results as JSON to stdout.                    | `false`       |
 | `--proto`              | Return results as binary serialized proto to stdout. | `false`       |
 | `--dump`               | Output the expected results, without a command.      | `false`       |
+| `--expected-failures`  | `.yaml` file containing list of expected failures    | None          |
 
 ## Components
 
@@ -42,7 +43,8 @@ collating the results.
 ### Executor
 
 Each target library should provide a _conformance test executor_, an executable
-responsible for reading a Protobuf-serialized [`TestConformanceRequest`][harness-proto] over
+responsible for reading a
+Protobuf-serialized [`TestConformanceRequest`][harness-proto] over
 `stdin`, executing the provided test case validations, and respond with a
 Protobuf-serialized [`TestConformanceResponse`][harness-proto] over `stdout`.
 
@@ -59,6 +61,43 @@ utilized to make concrete message _test cases_ which are organized into groups
 known as [_test suites_][suites]. Which tests are passed to the executor can be
 controlled through the `-suite` and `-case` flags mentioned above.
 
+### Expected Failures
+
+The conformance test harness can be configured to expect certain failures. This
+is useful for testing the behavior of a library when it encounters an invalid
+message. To use this feature, pass the `--expected-failures` flag with a path to
+a `.yaml` file containing a list of expected failures. The format of the file is
+as follows:
+
+```yaml
+<test_suite>:
+  - <test_case>
+```
+
+For example, if we wanted to test the behavior of a library when it encounters
+an invalid recursive message, we could create a file
+called `expected_failures.yaml` with the following contents:
+
+```yaml
+standard_constraints/map:
+  - recursive/invalid
+standard_constraints/well_known_types/duration:
+  - gte_lte/invalid/above
+  - lte/invalid
+  - not in/valid
+standard_constraints/well_known_types/timestamp:
+  - gte_lte/invalid/above
+  - lte/invalid
+```
+
+Then we could run the conformance tests with the following command:
+
+```shell
+.tmp/bin/protovalidate-conformance \
+  --expected-failures=expected_failures.yaml \
+  my-executor
+```
+
 [harness-proto]: /proto/protovalidate-testing/buf/validate/conformance/harness/harness.proto
 [cases-proto]: /proto/protovalidate-testing/buf/validate/conformance/cases
 [suites]: /tools/protovalidate-conformance/internal/cases

proto/protovalidate-testing/buf/validate/conformance/harness/results.proto

@@ -20,38 +20,65 @@ import "buf/validate/conformance/harness/harness.proto";
 import "google/protobuf/any.proto";
 import "google/protobuf/descriptor.proto";
 
+// ResultOptions are the options passed to the test runner to configure the
+// test run.
 message ResultOptions {
+  // The suite filter is a regex that matches against the suite name.
   string suite_filter = 1;
+  // The case filter is a regex that matches against the case name.
   string case_filter = 2;
+  // If the test runner should print verbose output.
   bool verbose = 3;
+  // If the violation type must be an exact match.
   bool strict = 4;
-
   // If the violation message must be an exact match.
   bool strict_message = 5;
-  // If the distinciton between runtime and compile time errors must be exact.
+  // If the distinction between runtime and compile time errors must be exact.
   bool strict_error = 6;
 }
 
+// A result is the result of a test run.
 message ResultSet {
+  // Count of successes.
   int32 successes = 1;
+  // Count of failures.
   int32 failures = 2;
+  // List of suite results.
   repeated SuiteResults suites = 3;
-
+  // Options used to generate this result.
   ResultOptions options = 4;
+  // Count of expected failures.
+  int32 expected_failures = 5;
 }
 
+// A suite result is a single test suite result.
 message SuiteResults {
+  // The suite name.
   string name = 1;
+  // Count of successes.
   int32 successes = 2;
+  // Count of failures.
   int32 failures = 3;
+  // List of case results.
   repeated CaseResult cases = 4;
+  // The file descriptor set used to generate this result.
   google.protobuf.FileDescriptorSet fdset = 5;
+  // Count of expected failures.
+  int32 expected_failures = 6;
 }
 
+// A case result is a single test case result.
 message CaseResult {
+  // The case name.
   string name = 1;
+  // Success state of the test case. True if the test case succeeded.
   bool success = 2;
+  // The expected result.
   TestResult wanted = 3;
+  // The actual result.
   TestResult got = 4;
+  // The input used to invoke the test case.
   google.protobuf.Any input = 5;
+  // Denotes if the test is expected to fail. True, if the test case was expected to fail.
+  bool expected_failure = 6;
 }