diff --git a/Cargo.lock b/Cargo.lock index faa025eb..2a907265 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,6 +216,7 @@ dependencies = [ "cfg-if", "getrandom", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -502,6 +503,21 @@ dependencies = [ "which", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "2.5.0" @@ -624,6 +640,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytecount" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" + [[package]] name = "bytes" version = "1.6.0" @@ -959,6 +981,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.1.0" @@ -1032,6 +1065,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1399,6 +1442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", + "futures-channel", "futures-util", "http 1.1.0", "http-body", @@ -1406,6 +1450,9 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -1453,7 +1500,7 @@ dependencies = [ "socket2", "widestring", "windows-sys 0.48.0", - "winreg", + "winreg 0.50.0", ] [[package]] @@ -1468,6 +1515,15 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "iso8601" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924e5d73ea28f59011fec52a0d12185d496a9b075d360657aed2a5707f701153" +dependencies = [ + "nom", +] + [[package]] name = "itertools" version = "0.12.1" @@ -1521,6 +1577,36 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonschema" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0afd06142c9bcb03f4a8787c77897a87b6be9c4918f1946c33caa714c27578" +dependencies = [ + "ahash 0.8.11", + "anyhow", + "base64 0.22.1", + "bytecount", + "clap", + "fancy-regex", + "fraction", + "getrandom", + "iso8601", + "itoa", + "memchr", + "num-cmp", + "once_cell", + "parking_lot", + "percent-encoding", + "regex", + "reqwest", + "serde", + "serde_json", + "time", + "url", + "uuid", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -1745,12 +1831,82 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1805,6 +1961,7 @@ dependencies = [ "hyper", "hyper-util", "jemallocator", + "jsonschema", "lazy_static", "libc", "native-tls", @@ -2194,6 +2351,42 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "reqwest" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "tokio", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.52.0", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -3062,6 +3255,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.92" @@ -3091,6 +3296,16 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "4.4.2" @@ -3289,6 +3504,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index ffb723ef..ad2f06c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ assert_cmd = "2.0.2" axum = { version = "0.7", features = ["http2"] } bytes = "1.0" float-cmp = "0.9.0" +jsonschema = "0.18.0" lazy_static = "1.4.0" predicates = "3.1.0" regex = "1.9.6" diff --git a/README.md b/README.md index ef348f28..e08d18de 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,11 @@ Options: -V, --version Print version ``` +# JSON output + +`oha` prints JSON output when `-j` option is set. +The schema of JSON output is defined in [schema.json](./schema.json). + # Benchmark ## Performance Comparison diff --git a/schema.json b/schema.json new file mode 100644 index 00000000..d9707b72 --- /dev/null +++ b/schema.json @@ -0,0 +1,357 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "JSON schema for the output of the `oha -j`", + "type": "object", + "properties": { + "summary": { + "description": "Important statistics", + "type": "object", + "properties": { + "successRate": { + "description": "The number of success requests / All requests which isn't includes deadline", + "type": "number" + }, + "total": { + "description": "Total duration in seconds", + "type": "number" + }, + "slowest": { + "description": "The slowest request duration in seconds", + "type": "number" + }, + "fastest": { + "description": "The fastest request duration in seconds", + "type": "number" + }, + "average": { + "description": "The average request duration in seconds", + "type": "number" + }, + "requestsPerSec": { + "description": "The number of requests per second", + "type": "number" + }, + "totalData": { + "description": "Total data of HTTP bodies in bytes", + "type": "integer" + }, + "sizePerRequest": { + "description": "The average size of HTTP bodies in bytes", + "type": "integer" + }, + "sizePerSec": { + "description": "The average size of HTTP bodies per second in bytes", + "type": "number" + } + }, + "required": [ + "successRate", + "total", + "slowest", + "fastest", + "average", + "requestsPerSec", + "totalData", + "sizePerRequest", + "sizePerSec" + ] + }, + "responseTimeHistogram": { + "description": "The histogram of response time in seconds. The key is the response time in seconds and the value is the number of requests", + "type": "object", + "additionalProperties": { + "string": "integer" + } + }, + "latencyPercentiles": { + "description": "The latency percentiles in seconds", + "type": "object", + "properties": { + "p10": { + "type": "number" + }, + "p25": { + "type": "number" + }, + "p50": { + "type": "number" + }, + "p75": { + "type": "number" + }, + "p90": { + "type": "number" + }, + "p95": { + "type": "number" + }, + "p99": { + "type": "number" + }, + "p99.9": { + "type": "number" + }, + "p99.99": { + "type": "number" + } + }, + "required": [ + "p10", + "p25", + "p50", + "p75", + "p90", + "p95", + "p99", + "p99.9", + "p99.99" + ] + }, + "responseTimeHistogramSuccessful": { + "description": "Only present if `--stats-success-breakdown` argument is passed. The histogram of response time in seconds for successful requests. The key is the response time in seconds and the value is the number of requests", + "type": "object", + "additionalProperties": { + "string": "integer" + } + }, + "latencyPercentileSuccessful": { + "description": "Only present if `--stats-success-breakdown` argument is passed. The latency percentiles in seconds for successful requests", + "type": "object", + "properties": { + "p10": { + "type": "number" + }, + "p25": { + "type": "number" + }, + "p50": { + "type": "number" + }, + "p75": { + "type": "number" + }, + "p90": { + "type": "number" + }, + "p95": { + "type": "number" + }, + "p99": { + "type": "number" + }, + "p99.9": { + "type": "number" + }, + "p99.99": { + "type": "number" + } + }, + "required": [ + "p10", + "p25", + "p50", + "p75", + "p90", + "p95", + "p99", + "p99.9", + "p99.99" + ] + }, + "responseTimeHistogramNotSuccessful": { + "description": "Only present if `--stats-success-breakdown` argument is passed. The histogram of response time in seconds for not successful requests. The key is the response time in seconds and the value is the number of requests", + "type": "object", + "additionalProperties": { + "string": "integer" + } + }, + "latencyPercentileNotSuccessful": { + "description": "Only present if `--stats-success-breakdown` argument is passed. The latency percentiles in seconds for not successful requests", + "type": "object", + "properties": { + "p10": { + "type": "number" + }, + "p25": { + "type": "number" + }, + "p50": { + "type": "number" + }, + "p75": { + "type": "number" + }, + "p90": { + "type": "number" + }, + "p95": { + "type": "number" + }, + "p99": { + "type": "number" + }, + "p99.9": { + "type": "number" + }, + "p99.99": { + "type": "number" + } + }, + "required": [ + "p10", + "p25", + "p50", + "p75", + "p90", + "p95", + "p99", + "p99.9", + "p99.99" + ] + }, + "rps": { + "description": "The statistics for requests per second. Note: the way of calculating rps over time isn't obvious, see source code for details.", + "type": "object", + "properties": { + "mean": { + "type": "number" + }, + "stddev": { + "type": [ + "number", + "null" + ] + }, + "max": { + "type": "number" + }, + "min": { + "type": "number" + }, + "percentiles": { + "type": "object", + "properties": { + "p10": { + "type": "number" + }, + "p25": { + "type": "number" + }, + "p50": { + "type": "number" + }, + "p75": { + "type": "number" + }, + "p90": { + "type": "number" + }, + "p95": { + "type": "number" + }, + "p99": { + "type": "number" + }, + "p99.9": { + "type": "number" + }, + "p99.99": { + "type": "number" + } + }, + "required": [ + "p10", + "p25", + "p50", + "p75", + "p90", + "p95", + "p99", + "p99.9", + "p99.99" + ] + } + }, + "required": [ + "mean", + "stddev", + "max", + "min", + "percentiles" + ] + }, + "details": { + "description": "The details of connection time. Note: `oha` uses keep-alive connections in default. So, the connection time may added only for the first request.", + "type": "object", + "properties": { + "DNSDialup": { + "description": "The time of DNS resolution + TCP handshake in seconds", + "type": "object", + "properties": { + "average": { + "type": "number" + }, + "fastest": { + "type": "number" + }, + "slowest": { + "type": "number" + } + }, + "required": [ + "average", + "fastest", + "slowest" + ] + }, + "DNSLookup": { + "description": "The time of DNS resolution in seconds", + "type": "object", + "properties": { + "average": { + "type": "number" + }, + "fastest": { + "type": "number" + }, + "slowest": { + "type": "number" + } + }, + "required": [ + "average", + "fastest", + "slowest" + ] + } + }, + "required": [ + "DNSDialup", + "DNSLookup" + ] + }, + "statusCodeDistribution": { + "description": "The distribution of status codes. The key is the status code and the value is the number of requests", + "type": "object", + "additionalProperties": { + "string": "integer" + } + }, + "errorDistribution": { + "description": "The distribution of errors. The key is the error message and the value is the number of errors. Note: the error message is from internal libraries so the detail may change in future.", + "type": "object", + "additionalProperties": { + "string": "integer" + } + } + }, + "required": [ + "summary", + "responseTimeHistogram", + "latencyPercentiles", + "rps", + "details", + "statusCodeDistribution", + "errorDistribution" + ] +} \ No newline at end of file diff --git a/src/printer.rs b/src/printer.rs index 24ffea9b..7500bf33 100644 --- a/src/printer.rs +++ b/src/printer.rs @@ -1,5 +1,5 @@ use crate::result_data::ResultData; -use average::{Max, Variance}; +use average::{Max, Min, Variance}; use byte_unit::Byte; use crossterm::style::{StyledContent, Stylize}; use hyper::http::{self, StatusCode}; @@ -161,6 +161,7 @@ fn print_json( mean: f64, stddev: f64, max: f64, + min: f64, percentiles: BTreeMap, } @@ -310,6 +311,7 @@ fn print_json( mean: variance.mean(), stddev: variance.sample_variance().sqrt(), max: rps.iter().collect::().max(), + min: rps.iter().collect::().min(), percentiles: rps_percentiles, }; diff --git a/tests/tests.rs b/tests/tests.rs index dad2a687..f2a6b563 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -703,8 +703,8 @@ async fn test_unix_socket() { rx.try_recv().unwrap(); } -#[tokio::test] -async fn test_google() { +#[test] +fn test_google() { Command::cargo_bin("oha") .unwrap() .args(["-n", "1", "--no-tui"]) @@ -713,3 +713,65 @@ async fn test_google() { .success() .stdout(predicates::str::contains("[200] 1 responses")); } + +#[tokio::test] +async fn test_json_schema() { + let app = Router::new().route("/", get(|| async move { "Hello World" })); + + let (listener, port) = bind_port().await; + tokio::spawn(async { axum::serve(listener, app).await }); + + const SCHEMA: &str = include_str!("../schema.json"); + let schema_value: serde_json::Value = serde_json::from_str(SCHEMA).unwrap(); + let schema = jsonschema::JSONSchema::compile(&schema_value).unwrap(); + + let output_json: String = String::from_utf8( + tokio::task::spawn_blocking(move || { + Command::cargo_bin("oha") + .unwrap() + .args(["-n", "10", "--no-tui", "-j"]) + .arg(format!("http://127.0.0.1:{port}/")) + .assert() + .get_output() + .stdout + .clone() + }) + .await + .unwrap(), + ) + .unwrap(); + + let output_json_stats_success_breakdown: String = String::from_utf8( + tokio::task::spawn_blocking(move || { + Command::cargo_bin("oha") + .unwrap() + .args(["-n", "10", "--no-tui", "-j", "--stats-success-breakdown"]) + .arg(format!("http://127.0.0.1:{port}/")) + .assert() + .get_output() + .stdout + .clone() + }) + .await + .unwrap(), + ) + .unwrap(); + + let value: serde_json::Value = serde_json::from_str(&output_json).unwrap(); + let value_stats_success_breakdown: serde_json::Value = + serde_json::from_str(&output_json_stats_success_breakdown).unwrap(); + + if let Err(errors) = schema.validate(&value) { + for error in errors { + eprintln!("{}", error); + } + panic!("JSON schema validation failed\n{output_json}"); + } + + if let Err(errors) = schema.validate(&value_stats_success_breakdown) { + for error in errors { + eprintln!("{}", error); + } + panic!("JSON schema validation failed\n{output_json_stats_success_breakdown}"); + }/* ?? */; +}