From 97b092d5c74e6b99076d900640bb12543bb691bc Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Mon, 30 Dec 2024 10:44:56 -0800 Subject: [PATCH 1/2] support running a tests in batches in a single process --- lib/parallel_tests/cli.rb | 18 +++++++++++++++++- spec/integration_spec.rb | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 37f86e25..adee7cd7 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -109,7 +109,22 @@ def run_tests_in_parallel(num_processes, options) end def run_tests(group, process_number, num_processes, options) - @runner.run_tests(group, process_number, num_processes, options) + if (limit = options[:test_file_limit]) + # TODO: will have some bugs with summarizing results and last process + results = group.each_slice(limit).map do |slice| + @runner.run_tests(slice, process_number, num_processes, options) + end + result = results[0] + results[1..].each do |res| + result[:stdout] = result[:stdout].to_s + res[:stdout].to_s + result[:exit_status] = [res[:exit_status], result[:exit_status]].max + # adding all files back in, not using original cmd to show what was actually run + result[:command] |= res[:command] + end + result + else + @runner.run_tests(group, process_number, num_processes, options) + end end def reprint_output(result, lockfile) @@ -286,6 +301,7 @@ def parse_options!(argv) opts.on("--unknown-runtime SECONDS", Float, "Use given number as unknown runtime (otherwise use average time)") { |time| options[:unknown_runtime] = time } opts.on("--first-is-1", "Use \"1\" as TEST_ENV_NUMBER to not reuse the default test environment") { options[:first_is_1] = true } opts.on("--fail-fast", "Stop all groups when one group fails (best used with --test-options '--fail-fast' if supported") { options[:fail_fast] = true } + opts.on("--test-file-limit LIMIT", Integer, "Limit to this number of files per test run by batching (for windows set to ~100 to stay below 8192 max command limit, might have bugs from reusing test-env-number and summarizing partial results)") { |limit| options[:test_file_limit] = limit } opts.on("--verbose", "Print debug output") { options[:verbose] = true } opts.on("--verbose-command", "Combines options --verbose-process-command and --verbose-rerun-command") { options.merge! verbose_process_command: true, verbose_rerun_command: true } opts.on("--verbose-process-command", "Print the command that will be executed by each process before it begins") { options[:verbose_process_command] = true } diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 468d0aaf..44a08de0 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -711,4 +711,27 @@ class A < Spinach::FeatureSteps expect(result).to_not include("Should not get here") end end + + describe "--test-file-limit" do + let(:test_count) { 3 } + before do + test_count.times do |i| + write "spec/x#{i}_spec.rb", "puts %(TEST#{i}-\#{ENV['TEST_ENV_NUMBER']}-\#{Process.pid})" + end + end + + it "runs in batches" do + result = run_tests ["spec"], type: 'rspec', add: ['--test-file-limit', '1', '--first-is-1', '-n', '2'] + expect(result.scan(/TEST\d-\d/).sort).to eq(["TEST0-1", "TEST1-2", "TEST2-1"]) + pids = result.scan(/TEST\d-\d-(\d+)/).flatten.uniq + expect(pids.size).to eq test_count # did not run 2 tests in the same process + end + + it "does not run in batches when above limit" do + result = run_tests ["spec"], type: 'rspec', add: ['--test-file-limit', '2', '--first-is-1', '-n', '2'] + expect(result.scan(/TEST\d-\d/).sort).to eq(["TEST0-1", "TEST1-2", "TEST2-1"]) + pids = result.scan(/TEST\d-\d-(\d+)/).flatten.uniq + expect(pids.size).to eq 2 + end + end end From 4496509c3d8a814b24f9908295fb80a80e8e5943 Mon Sep 17 00:00:00 2001 From: Michael Grosser Date: Thu, 2 Jan 2025 14:33:15 -0800 Subject: [PATCH 2/2] fix win --- spec/integration_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb index 44a08de0..a2dce5a9 100644 --- a/spec/integration_spec.rb +++ b/spec/integration_spec.rb @@ -716,21 +716,21 @@ class A < Spinach::FeatureSteps let(:test_count) { 3 } before do test_count.times do |i| - write "spec/x#{i}_spec.rb", "puts %(TEST#{i}-\#{ENV['TEST_ENV_NUMBER']}-\#{Process.pid})" + write "spec/x#{i}_spec.rb", "puts %(TEST-\#{ENV['TEST_ENV_NUMBER']}-\#{Process.pid})" end end it "runs in batches" do result = run_tests ["spec"], type: 'rspec', add: ['--test-file-limit', '1', '--first-is-1', '-n', '2'] - expect(result.scan(/TEST\d-\d/).sort).to eq(["TEST0-1", "TEST1-2", "TEST2-1"]) - pids = result.scan(/TEST\d-\d-(\d+)/).flatten.uniq + expect(result.scan(/TEST-\d/).sort).to eq(["TEST-1", "TEST-1", "TEST-2"]) + pids = result.scan(/TEST-\d-(\d+)/).flatten.uniq expect(pids.size).to eq test_count # did not run 2 tests in the same process end it "does not run in batches when above limit" do result = run_tests ["spec"], type: 'rspec', add: ['--test-file-limit', '2', '--first-is-1', '-n', '2'] - expect(result.scan(/TEST\d-\d/).sort).to eq(["TEST0-1", "TEST1-2", "TEST2-1"]) - pids = result.scan(/TEST\d-\d-(\d+)/).flatten.uniq + expect(result.scan(/TEST-\d/).sort).to eq(["TEST-1", "TEST-1", "TEST-2"]) + pids = result.scan(/TEST-\d-(\d+)/).flatten.uniq expect(pids.size).to eq 2 end end