From 4f12b9be1eab8678fbb00f9ea2a9afdcbfa1012d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Freddy=20Rom=C3=A1n?= Date: Sat, 30 Apr 2022 11:35:52 -0700 Subject: [PATCH 1/7] rfc --- common/literalinput.go | 7 ++++- common/problemsettings.go | 3 +++ runner/runner.go | 55 ++++++++++++++++++++++++++++++--------- runner/sandbox.go | 16 +++++++++--- runner/sandbox_test.go | 1 + 5 files changed, 66 insertions(+), 16 deletions(-) diff --git a/common/literalinput.go b/common/literalinput.go index bea9b61..16ab000 100644 --- a/common/literalinput.go +++ b/common/literalinput.go @@ -143,7 +143,8 @@ var ( MemoryLimit: base.Gibibyte + base.Gibibyte/2, OverallWallTimeLimit: base.Duration(time.Duration(10) * time.Second), ExtraWallTime: base.Duration(0), - OutputLimit: base.Byte(2) * base.Mebibyte, + OutputLimit: base.Byte(100) * base.Mebibyte, + OverallOutputLimit: base.Byte(100) * base.Mebibyte, } DefaultLiteralValidatorSettings = LiteralValidatorSettings{ @@ -306,6 +307,10 @@ func NewLiteralInputFactory( input.Limits.OutputLimit, DefaultLiteralLimitSettings.OutputLimit, ) + settings.Limits.OverallOutputLimit = base.MinBytes( + input.Limits.OverallOutputLimit, + DefaultLiteralLimitSettings.OverallOutputLimit, + ) } else { settings.Limits = DefaultLiteralLimitSettings } diff --git a/common/problemsettings.go b/common/problemsettings.go index e9b1dca..807f7e5 100644 --- a/common/problemsettings.go +++ b/common/problemsettings.go @@ -21,6 +21,7 @@ type LimitsSettings struct { ExtraWallTime base.Duration MemoryLimit base.Byte OutputLimit base.Byte + OverallOutputLimit base.Byte OverallWallTimeLimit base.Duration TimeLimit base.Duration } @@ -189,6 +190,7 @@ var ( OutputLimit: base.Byte(10) * base.Kibibyte, OverallWallTimeLimit: base.Duration(time.Duration(5) * time.Second), TimeLimit: base.Duration(time.Duration(1) * time.Second), + OverallOutputLimit: base.Byte(100) * base.Mebibyte, } // DefaultLimits specifies the default limits for a problem. @@ -198,6 +200,7 @@ var ( OutputLimit: base.Byte(10) * base.Kibibyte, OverallWallTimeLimit: base.Duration(time.Duration(1) * time.Minute), TimeLimit: base.Duration(time.Duration(1) * time.Second), + OverallOutputLimit: base.Byte(100) * base.Mebibyte, } ) diff --git a/runner/runner.go b/runner/runner.go index 9f77b62..8cb404c 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -142,17 +142,18 @@ func (g *GroupResult) UnmarshalJSON(data []byte) error { // A RunResult represents the results of a run. type RunResult struct { - Verdict string `json:"verdict"` - CompileError *string `json:"compile_error,omitempty"` - CompileMeta map[string]RunMetadata `json:"compile_meta"` - Score *big.Rat `json:"score"` - ContestScore *big.Rat `json:"contest_score"` - MaxScore *big.Rat `json:"max_score"` - Time float64 `json:"time"` - WallTime float64 `json:"wall_time"` - Memory base.Byte `json:"memory"` - JudgedBy string `json:"judged_by,omitempty"` - Groups []GroupResult `json:"groups"` + Verdict string `json:"verdict"` + CompileError *string `json:"compile_error,omitempty"` + CompileMeta map[string]RunMetadata `json:"compile_meta"` + Score *big.Rat `json:"score"` + ContestScore *big.Rat `json:"contest_score"` + MaxScore *big.Rat `json:"max_score"` + Time float64 `json:"time"` + WallTime float64 `json:"wall_time"` + Memory base.Byte `json:"memory"` + OverallOutput base.Byte `json:"total_output"` + JudgedBy string `json:"judged_by,omitempty"` + Groups []GroupResult `json:"groups"` } // NewRunResult returns a new RunResult. @@ -395,6 +396,7 @@ func parseOutputOnlyFile( ) (map[string]outputOnlyFile, error) { dataURL, err := dataurl.DecodeString(data) result := make(map[string]outputOnlyFile) + overallOutput := base.Byte(0) if err != nil { // |data| is not a dataurl. Try just returning the data as an Entry. ctx.Log.Info( @@ -460,6 +462,18 @@ func parseOutputOnlyFile( result[fileName] = outputOnlyFile{"", true} continue } + if overallOutput > settings.Limits.OverallOutputLimit { + ctx.Log.Info( + "Output-only overall size limit has been exceeded. Generating empty file", + map[string]interface{}{ + "name": f.FileHeader.Name, + "overall output": overallOutput, + "limit": settings.Limits.OverallOutputLimit, + }, + ) + result[fileName] = outputOnlyFile{"", true} + continue + } rc, err := f.Open() if err != nil { ctx.Log.Info( @@ -485,6 +499,7 @@ func parseOutputOnlyFile( continue } result[fileName] = outputOnlyFile{buf.String(), false} + overallOutput += base.Byte(buf.Len()) } return result, nil } @@ -974,6 +989,18 @@ func Grade( runMeta = &RunMetadata{ Verdict: "TLE", } + } else if runResult.OverallOutput > settings.Limits.OverallOutputLimit { + ctx.Log.Debug( + "Not even running since the overall output limit has been exceeded", + map[string]interface{}{ + "case": caseData.Name, + "overall output": runResult.OverallOutput, + "limit": settings.Limits.OverallOutputLimit, + }, + ) + runMeta = &RunMetadata{ + Verdict: "OLE", + } } else if run.Language == "cat" { outName := fmt.Sprintf("%s.out", caseData.Name) errName := fmt.Sprintf("%s.err", caseData.Name) @@ -1146,6 +1173,7 @@ func Grade( var totalTime float64 var totalWallTime float64 var totalMemory base.Byte + var totalOutput base.Byte for i := 0; i < regularBinaryCount; i++ { intermediateResult := <-metaChan generatedFiles = append(generatedFiles, intermediateResult.generatedFiles...) @@ -1171,7 +1199,8 @@ func Grade( totalWallTime, intermediateResult.runMeta.WallTime, ) - totalMemory += base.MaxBytes(totalMemory, intermediateResult.runMeta.Memory) + totalMemory = base.MaxBytes(totalMemory, intermediateResult.runMeta.Memory) + totalOutput = base.MaxBytes(totalOutput, intermediateResult.runMeta.OutputSize) } } close(metaChan) @@ -1180,6 +1209,7 @@ func Grade( chosenMetadata.Time = totalTime chosenMetadata.WallTime = totalWallTime chosenMetadata.Memory = totalMemory + chosenMetadata.OutputSize = totalOutput runMeta = mergeVerdict(ctx, &chosenMetadata, parentMetadata) } @@ -1187,6 +1217,7 @@ func Grade( runResult.Time += runMeta.Time runResult.WallTime += runMeta.WallTime runResult.Memory = base.MaxBytes(runResult.Memory, runMeta.Memory) + runResult.OverallOutput += runMeta.OutputSize // TODO: change CaseResult to split original metadatas and final metadata caseResults[j] = CaseResult{ diff --git a/runner/sandbox.go b/runner/sandbox.go index e5e280c..a4f6b52 100644 --- a/runner/sandbox.go +++ b/runner/sandbox.go @@ -79,19 +79,21 @@ type RunMetadata struct { SystemTime float64 `json:"sys_time"` WallTime float64 `json:"wall_time"` Memory base.Byte `json:"memory"` + OutputSize base.Byte `json:"output_size"` Signal *string `json:"signal,omitempty"` Syscall *string `json:"syscall,omitempty"` } func (m *RunMetadata) String() string { metadata := fmt.Sprintf( - "{Verdict: %s, ExitStatus: %d, Time: %.3fs, SystemTime: %.3fs, WallTime: %.3fs, Memory: %.3fMiB", + "{Verdict: %s, ExitStatus: %d, Time: %.3fs, SystemTime: %.3fs, WallTime: %.3fs, Memory: %.3fMiB, OutputSize: %.3fMiB", m.Verdict, m.ExitStatus, m.Time, m.SystemTime, m.WallTime, m.Memory.Mebibytes(), + m.OutputSize.Mebibytes(), ) if m.Signal != nil { metadata += fmt.Sprintf(", Signal: %s", *m.Signal) @@ -220,7 +222,7 @@ func (o *OmegajailSandbox) Compile( }, err } defer metaFd.Close() - metadata, err := parseMetaFile(ctx, nil, lang, metaFd, nil, false) + metadata, err := parseMetaFile(ctx, nil, lang, metaFd, &outputFile, nil, false) if lang == "java" && metadata.Verdict == "OK" { classPath := path.Join(chdir, fmt.Sprintf("%s.class", target)) @@ -358,7 +360,7 @@ func (o *OmegajailSandbox) Run( }, err } defer metaFd.Close() - return parseMetaFile(ctx, limits, lang, metaFd, &errorFile, lang == "c") + return parseMetaFile(ctx, limits, lang, metaFd, &outputFile, &errorFile, lang == "c") } func invokeOmegajail(ctx *common.Context, omegajailRoot string, omegajailParams []string, errorFile string) { @@ -433,6 +435,7 @@ func parseMetaFile( limits *common.LimitsSettings, lang string, metaFile io.Reader, + outputFilePath *string, errorFilePath *string, allowNonZeroExitCode bool, ) (*RunMetadata, error) { @@ -524,6 +527,13 @@ func parseMetaFile( meta.Memory = limits.MemoryLimit } + if outputFilePath != nil { + outputFileStat, err := os.Stat(*outputFilePath) + if err == nil { + meta.OutputSize = base.Byte(outputFileStat.Size()) + } + } + return meta, nil } diff --git a/runner/sandbox_test.go b/runner/sandbox_test.go index 2a13381..e0924ae 100644 --- a/runner/sandbox_test.go +++ b/runner/sandbox_test.go @@ -138,6 +138,7 @@ func TestParseMetaFile(t *testing.T) { te.lang, bytes.NewBufferString(te.contents), nil, + nil, te.lang == "c", ) if err != nil { From 2e7822417493bdbfee39a47a08badf7e6c1276b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Freddy=20Rom=C3=A1n?= Date: Sat, 30 Apr 2022 21:27:42 -0700 Subject: [PATCH 2/7] OLE test --- common/context.go | 4 ++ common/literalinput.go | 5 -- common/problemsettings.go | 3 - runner/runner.go | 8 +-- runner/runner_test.go | 135 ++++++++++++++++++++++++++++++++++++++ runner/sandbox.go | 2 +- 6 files changed, 144 insertions(+), 13 deletions(-) diff --git a/common/context.go b/common/context.go index 7f0a3bf..f3ba707 100644 --- a/common/context.go +++ b/common/context.go @@ -90,6 +90,8 @@ type RunnerConfig struct { RuntimePath string CompileTimeLimit base.Duration CompileOutputLimit base.Byte + HardMemoryLimit base.Byte + OverallOutputLimit base.Byte OmegajailRoot string PreserveFiles bool } @@ -202,6 +204,8 @@ var defaultConfig = Config{ GraderURL: "https://omegaup.com:11302", CompileTimeLimit: base.Duration(time.Duration(30) * time.Second), CompileOutputLimit: base.Byte(10) * base.Mebibyte, + HardMemoryLimit: base.Byte(640) * base.Mebibyte, + OverallOutputLimit: base.Byte(100) * base.Mebibyte, OmegajailRoot: "/var/lib/omegajail", PreserveFiles: false, }, diff --git a/common/literalinput.go b/common/literalinput.go index 16ab000..ee5f607 100644 --- a/common/literalinput.go +++ b/common/literalinput.go @@ -144,7 +144,6 @@ var ( OverallWallTimeLimit: base.Duration(time.Duration(10) * time.Second), ExtraWallTime: base.Duration(0), OutputLimit: base.Byte(100) * base.Mebibyte, - OverallOutputLimit: base.Byte(100) * base.Mebibyte, } DefaultLiteralValidatorSettings = LiteralValidatorSettings{ @@ -307,10 +306,6 @@ func NewLiteralInputFactory( input.Limits.OutputLimit, DefaultLiteralLimitSettings.OutputLimit, ) - settings.Limits.OverallOutputLimit = base.MinBytes( - input.Limits.OverallOutputLimit, - DefaultLiteralLimitSettings.OverallOutputLimit, - ) } else { settings.Limits = DefaultLiteralLimitSettings } diff --git a/common/problemsettings.go b/common/problemsettings.go index 807f7e5..e9b1dca 100644 --- a/common/problemsettings.go +++ b/common/problemsettings.go @@ -21,7 +21,6 @@ type LimitsSettings struct { ExtraWallTime base.Duration MemoryLimit base.Byte OutputLimit base.Byte - OverallOutputLimit base.Byte OverallWallTimeLimit base.Duration TimeLimit base.Duration } @@ -190,7 +189,6 @@ var ( OutputLimit: base.Byte(10) * base.Kibibyte, OverallWallTimeLimit: base.Duration(time.Duration(5) * time.Second), TimeLimit: base.Duration(time.Duration(1) * time.Second), - OverallOutputLimit: base.Byte(100) * base.Mebibyte, } // DefaultLimits specifies the default limits for a problem. @@ -200,7 +198,6 @@ var ( OutputLimit: base.Byte(10) * base.Kibibyte, OverallWallTimeLimit: base.Duration(time.Duration(1) * time.Minute), TimeLimit: base.Duration(time.Duration(1) * time.Second), - OverallOutputLimit: base.Byte(100) * base.Mebibyte, } ) diff --git a/runner/runner.go b/runner/runner.go index 8cb404c..633ed30 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -462,13 +462,13 @@ func parseOutputOnlyFile( result[fileName] = outputOnlyFile{"", true} continue } - if overallOutput > settings.Limits.OverallOutputLimit { + if overallOutput > ctx.Config.Runner.OverallOutputLimit { ctx.Log.Info( "Output-only overall size limit has been exceeded. Generating empty file", map[string]interface{}{ "name": f.FileHeader.Name, "overall output": overallOutput, - "limit": settings.Limits.OverallOutputLimit, + "limit": ctx.Config.Runner.OverallOutputLimit, }, ) result[fileName] = outputOnlyFile{"", true} @@ -989,13 +989,13 @@ func Grade( runMeta = &RunMetadata{ Verdict: "TLE", } - } else if runResult.OverallOutput > settings.Limits.OverallOutputLimit { + } else if runResult.OverallOutput > ctx.Config.Runner.OverallOutputLimit { ctx.Log.Debug( "Not even running since the overall output limit has been exceeded", map[string]interface{}{ "case": caseData.Name, "overall output": runResult.OverallOutput, - "limit": settings.Limits.OverallOutputLimit, + "limit": ctx.Config.Runner.OverallOutputLimit, }, ) runMeta = &RunMetadata{ diff --git a/runner/runner_test.go b/runner/runner_test.go index 182dc30..356b4e8 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1227,6 +1227,141 @@ func TestGradeLowMemOmegajail(t *testing.T) { } } +func TestGradeOLEOmegajail(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + omegajail := getSandbox() + if !omegajail.Supported() { + t.Skip("omegajail omegajail not supported") + } + + ctx, err := newRunnerContext(t) + if err != nil { + t.Fatalf("RunnerContext creation failed with %q", err) + } + defer ctx.Close() + if !ctx.Config.Runner.PreserveFiles { + defer os.RemoveAll(ctx.Config.Runner.RuntimePath) + } + + // Artificially low limit for the test + ctx.Config.Runner.OverallOutputLimit = base.Byte(3) + + inputManager := common.NewInputManager(ctx) + AplusB, err := common.NewLiteralInputFactory( + &common.LiteralInput{ + Cases: map[string]*common.LiteralCaseSettings{ + "0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)}, + "1.0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)}, + "1.1": {Input: "2 3", ExpectedOutput: "5", Weight: big.NewRat(2, 1)}, + }, + Validator: &common.LiteralValidatorSettings{ + Name: common.ValidatorNameTokenNumeric, + }, + Limits: &common.LimitsSettings{ + TimeLimit: common.DefaultLiteralLimitSettings.TimeLimit, + MemoryLimit: 8 * 1024 * 1024, + OverallWallTimeLimit: common.DefaultLiteralLimitSettings.OverallWallTimeLimit, + ExtraWallTime: common.DefaultLiteralLimitSettings.ExtraWallTime, + OutputLimit: common.DefaultLiteralLimitSettings.OutputLimit, + }, + }, + ctx.Config.Runner.RuntimePath, + common.LiteralPersistRunner, + ) + if err != nil { + t.Fatalf("Failed to create Input: %q", err) + } + + inputRef, err := inputManager.Add(AplusB.Hash(), AplusB) + if err != nil { + t.Fatalf("Failed to open problem: %q", err) + } + defer inputRef.Release() + + runtests := []runnerTestCase{ + { + "c11-gcc", + "#include \nint main() { printf(\"3\\n\"); }", + big.NewRat(1, 1), + "OLE", + big.NewRat(1, 4), + expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, + map[string]expectedResult{ + "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + }, + }, + { + "cpp17-gcc", + "#include \nint main() { std::cout << \"3\\n\"; }", + big.NewRat(1, 1), + "OLE", + big.NewRat(1, 4), + expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, + map[string]expectedResult{ + "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + }, + }, + { + "pas", + `program Main; + begin + writeln ('3'); + end.`, + big.NewRat(1, 1), + "OLE", + big.NewRat(1, 4), + expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, + map[string]expectedResult{ + "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + }, + }, + } + for idx, rte := range runtests { + t.Run(fmt.Sprintf("%d/%s %s", idx, rte.language, rte.expectedVerdict), func(t *testing.T) { + results, err := Grade( + ctx, + &bytes.Buffer{}, + &common.Run{ + AttemptID: uint64(idx), + Language: rte.language, + InputHash: inputRef.Input.Hash(), + Source: rte.source, + MaxScore: rte.maxScore, + }, + inputRef.Input, + omegajail, + ) + if err != nil { + t.Fatalf("Failed to run %v: %q", rte, err) + } + if results.Verdict != rte.expectedVerdict { + t.Errorf( + "results.Verdict = %q, expected %q, test %v: %v", + results.Verdict, + rte.expectedVerdict, + idx, + rte, + ) + } + if results.Score.Cmp(rte.expectedScore) != 0 { + t.Errorf( + "results.Score = %s, expected %s", + results.Score.String(), + rte.expectedScore.String(), + ) + } + }) + } +} + func TestKarelGrade(t *testing.T) { for name, wrapper := range map[string]sandboxWrapper{ "fake": &fakeSandboxWrapper{}, diff --git a/runner/sandbox.go b/runner/sandbox.go index a4f6b52..efdca12 100644 --- a/runner/sandbox.go +++ b/runner/sandbox.go @@ -310,7 +310,7 @@ func (o *OmegajailSandbox) Run( } // "640MB should be enough for anybody" - hardLimit := base.MinBytes(base.Byte(640)*base.Mebibyte, limits.MemoryLimit) + hardLimit := base.MinBytes(ctx.Config.Runner.HardMemoryLimit, limits.MemoryLimit) params := []string{ "--homedir", chdir, From fc58b70d86b98ec452b62daf55e8f6a3ebdde22f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Freddy=20Rom=C3=A1n?= Date: Sat, 30 Apr 2022 21:39:46 -0700 Subject: [PATCH 3/7] better tests --- runner/runner_test.go | 247 ++++++++++++++++++++++-------------------- 1 file changed, 127 insertions(+), 120 deletions(-) diff --git a/runner/runner_test.go b/runner/runner_test.go index 356b4e8..267d96a 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1227,136 +1227,143 @@ func TestGradeLowMemOmegajail(t *testing.T) { } } -func TestGradeOLEOmegajail(t *testing.T) { - if testing.Short() { - t.Skip("skipping test in short mode.") - } - omegajail := getSandbox() - if !omegajail.Supported() { - t.Skip("omegajail omegajail not supported") - } +func TestGradeOLE(t *testing.T) { + for name, wrapper := range map[string]sandboxWrapper{ + "fake": &fakeSandboxWrapper{}, + "omegajail": &omegajailSandboxWrapper{omegajail: getSandbox()}, + } { + wrapper := wrapper + t.Run(name, func(t *testing.T) { + if testing.Short() && wrapper.name() == "OmegajailSandbox" { + t.Skip("skipping test in short mode.") + } + if !wrapper.supported() { + t.Skip(fmt.Sprintf("%s not supported", wrapper.name())) + } - ctx, err := newRunnerContext(t) - if err != nil { - t.Fatalf("RunnerContext creation failed with %q", err) - } - defer ctx.Close() - if !ctx.Config.Runner.PreserveFiles { - defer os.RemoveAll(ctx.Config.Runner.RuntimePath) - } + ctx, err := newRunnerContext(t) + if err != nil { + t.Fatalf("RunnerContext creation failed with %q", err) + } + defer ctx.Close() + if !ctx.Config.Runner.PreserveFiles { + defer os.RemoveAll(ctx.Config.Runner.RuntimePath) + } - // Artificially low limit for the test - ctx.Config.Runner.OverallOutputLimit = base.Byte(3) + // Artificially low limit for the test + ctx.Config.Runner.OverallOutputLimit = base.Byte(3) - inputManager := common.NewInputManager(ctx) - AplusB, err := common.NewLiteralInputFactory( - &common.LiteralInput{ - Cases: map[string]*common.LiteralCaseSettings{ - "0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)}, - "1.0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)}, - "1.1": {Input: "2 3", ExpectedOutput: "5", Weight: big.NewRat(2, 1)}, - }, - Validator: &common.LiteralValidatorSettings{ - Name: common.ValidatorNameTokenNumeric, - }, - Limits: &common.LimitsSettings{ - TimeLimit: common.DefaultLiteralLimitSettings.TimeLimit, - MemoryLimit: 8 * 1024 * 1024, - OverallWallTimeLimit: common.DefaultLiteralLimitSettings.OverallWallTimeLimit, - ExtraWallTime: common.DefaultLiteralLimitSettings.ExtraWallTime, - OutputLimit: common.DefaultLiteralLimitSettings.OutputLimit, - }, - }, - ctx.Config.Runner.RuntimePath, - common.LiteralPersistRunner, - ) - if err != nil { - t.Fatalf("Failed to create Input: %q", err) - } + inputManager := common.NewInputManager(ctx) + AplusB, err := common.NewLiteralInputFactory( + &common.LiteralInput{ + Cases: map[string]*common.LiteralCaseSettings{ + "0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)}, + "1.0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)}, + "1.1": {Input: "2 3", ExpectedOutput: "5", Weight: big.NewRat(2, 1)}, + }, + Validator: &common.LiteralValidatorSettings{ + Name: common.ValidatorNameTokenNumeric, + }, + Limits: &common.LimitsSettings{ + TimeLimit: common.DefaultLiteralLimitSettings.TimeLimit, + MemoryLimit: 8 * 1024 * 1024, + OverallWallTimeLimit: common.DefaultLiteralLimitSettings.OverallWallTimeLimit, + ExtraWallTime: common.DefaultLiteralLimitSettings.ExtraWallTime, + OutputLimit: common.DefaultLiteralLimitSettings.OutputLimit, + }, + }, + ctx.Config.Runner.RuntimePath, + common.LiteralPersistRunner, + ) + if err != nil { + t.Fatalf("Failed to create Input: %q", err) + } - inputRef, err := inputManager.Add(AplusB.Hash(), AplusB) - if err != nil { - t.Fatalf("Failed to open problem: %q", err) - } - defer inputRef.Release() + inputRef, err := inputManager.Add(AplusB.Hash(), AplusB) + if err != nil { + t.Fatalf("Failed to open problem: %q", err) + } + defer inputRef.Release() - runtests := []runnerTestCase{ - { - "c11-gcc", - "#include \nint main() { printf(\"3\\n\"); }", - big.NewRat(1, 1), - "OLE", - big.NewRat(1, 4), - expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, - map[string]expectedResult{ - "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, - }, - }, - { - "cpp17-gcc", - "#include \nint main() { std::cout << \"3\\n\"; }", - big.NewRat(1, 1), - "OLE", - big.NewRat(1, 4), - expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, - map[string]expectedResult{ - "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, - }, - }, - { - "pas", - `program Main; + runtests := []runnerTestCase{ + { + "c11-gcc", + "#include \nint main() { printf(\"3\\n\"); }", + big.NewRat(1, 1), + "OLE", + big.NewRat(1, 4), + expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, + map[string]expectedResult{ + "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + }, + }, + { + "cpp17-gcc", + "#include \nint main() { std::cout << \"3\\n\"; }", + big.NewRat(1, 1), + "OLE", + big.NewRat(1, 4), + expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, + map[string]expectedResult{ + "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + }, + }, + { + "pas", + `program Main; begin writeln ('3'); end.`, - big.NewRat(1, 1), - "OLE", - big.NewRat(1, 4), - expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, - map[string]expectedResult{ - "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, - }, - }, - } - for idx, rte := range runtests { - t.Run(fmt.Sprintf("%d/%s %s", idx, rte.language, rte.expectedVerdict), func(t *testing.T) { - results, err := Grade( - ctx, - &bytes.Buffer{}, - &common.Run{ - AttemptID: uint64(idx), - Language: rte.language, - InputHash: inputRef.Input.Hash(), - Source: rte.source, - MaxScore: rte.maxScore, + big.NewRat(1, 1), + "OLE", + big.NewRat(1, 4), + expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, + map[string]expectedResult{ + "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + }, }, - inputRef.Input, - omegajail, - ) - if err != nil { - t.Fatalf("Failed to run %v: %q", rte, err) - } - if results.Verdict != rte.expectedVerdict { - t.Errorf( - "results.Verdict = %q, expected %q, test %v: %v", - results.Verdict, - rte.expectedVerdict, - idx, - rte, - ) } - if results.Score.Cmp(rte.expectedScore) != 0 { - t.Errorf( - "results.Score = %s, expected %s", - results.Score.String(), - rte.expectedScore.String(), - ) + for idx, rte := range runtests { + t.Run(fmt.Sprintf("%s/%d/%s %s", wrapper.name(), idx, rte.language, rte.expectedVerdict), func(t *testing.T) { + results, err := Grade( + ctx, + &bytes.Buffer{}, + &common.Run{ + AttemptID: uint64(idx), + Language: rte.language, + InputHash: inputRef.Input.Hash(), + Source: rte.source, + MaxScore: rte.maxScore, + }, + inputRef.Input, + wrapper.sandbox(&rte), + ) + if err != nil { + t.Fatalf("Failed to run %v: %q", rte, err) + } + if results.Verdict != rte.expectedVerdict { + t.Errorf( + "results.Verdict = %q, expected %q, test %v: %v", + results.Verdict, + rte.expectedVerdict, + idx, + rte, + ) + } + if results.Score.Cmp(rte.expectedScore) != 0 { + t.Errorf( + "results.Score = %s, expected %s", + results.Score.String(), + rte.expectedScore.String(), + ) + } + }) } }) } From efe4c537a8680b398b699b4f6ae040ee7ac55f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Freddy=20Rom=C3=A1n?= Date: Sat, 30 Apr 2022 21:42:55 -0700 Subject: [PATCH 4/7] even better tests --- runner/runner_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/runner/runner_test.go b/runner/runner_test.go index 267d96a..cd6f7f1 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1328,6 +1328,27 @@ func TestGradeOLE(t *testing.T) { "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, }, }, + { + "cat", + "data:application/zip;base64,UEsDBAoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABwAMC5vdX" + + "RVVAkAA67WxFb8t4ZYdXgLAAEE6AMAAAToAwAAMwpQSwMECgAAAAAAhhE4StGeZ1UCAAAAAg" + + "AAAAcAHAAxLjAub3V0VVQJAAP8t4ZYCbiGWHV4CwABBOgDAAAE6AMAADMKUEsDBAoAAAAAAO" + + "eiUUhXOT0DAgAAAAIAAAAHABwAMS4xLm91dFVUCQADstbEVgm4hlh1eAsAAQToAwAABOgDAA" + + "A1ClBLAQIeAwoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABgAAAAAAAEAAAC0gQAAAAAwLm91dF" + + "VUBQADrtbEVnV4CwABBOgDAAAE6AMAAFBLAQIeAwoAAAAAAIYROErRnmdVAgAAAAIAAAAHAB" + + "gAAAAAAAEAAAC0gUEAAAAxLjAub3V0VVQFAAP8t4ZYdXgLAAEE6AMAAAToAwAAUEsBAh4DCg" + + "AAAAAA56JRSFc5PQMCAAAAAgAAAAcAGAAAAAAAAQAAALSBhAAAADEuMS5vdXRVVAUAA7LWxF" + + "Z1eAsAAQToAwAABOgDAABQSwUGAAAAAAMAAwDlAAAAxwAAAAAA", + big.NewRat(1, 1), + "OLE", + big.NewRat(1, 4), + expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, + map[string]expectedResult{ + "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + }, + }, } for idx, rte := range runtests { t.Run(fmt.Sprintf("%s/%d/%s %s", wrapper.name(), idx, rte.language, rte.expectedVerdict), func(t *testing.T) { From 4a71b7fd027d43a05b8316dc7432214072d530cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Freddy=20Rom=C3=A1n?= Date: Sat, 30 Apr 2022 22:08:01 -0700 Subject: [PATCH 5/7] ? --- runner/runner_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runner/runner_test.go b/runner/runner_test.go index cd6f7f1..88cdc81 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1296,7 +1296,7 @@ func TestGradeOLE(t *testing.T) { map[string]expectedResult{ "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE", OutputSize: base.Byte(2)}}}, }, }, { @@ -1309,7 +1309,7 @@ func TestGradeOLE(t *testing.T) { map[string]expectedResult{ "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE", OutputSize: base.Byte(2)}}}, }, }, { @@ -1325,7 +1325,7 @@ func TestGradeOLE(t *testing.T) { map[string]expectedResult{ "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE", OutputSize: base.Byte(2)}}}, }, }, { @@ -1346,7 +1346,7 @@ func TestGradeOLE(t *testing.T) { map[string]expectedResult{ "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE", OutputSize: base.Byte(2)}}}, }, }, } From ca0e5ac3bfedcd56c7a16bd59de9db8a730a1d91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Freddy=20Rom=C3=A1n?= Date: Sat, 30 Apr 2022 22:19:06 -0700 Subject: [PATCH 6/7] counting is hard --- runner/runner_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/runner/runner_test.go b/runner/runner_test.go index 88cdc81..a9b0f4e 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1294,9 +1294,9 @@ func TestGradeOLE(t *testing.T) { big.NewRat(1, 4), expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, map[string]expectedResult{ - "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE", OutputSize: base.Byte(2)}}}, + "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, }, }, { @@ -1307,9 +1307,9 @@ func TestGradeOLE(t *testing.T) { big.NewRat(1, 4), expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, map[string]expectedResult{ - "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE", OutputSize: base.Byte(2)}}}, + "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, }, }, { @@ -1325,7 +1325,7 @@ func TestGradeOLE(t *testing.T) { map[string]expectedResult{ "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE", OutputSize: base.Byte(2)}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, }, }, { @@ -1346,7 +1346,7 @@ func TestGradeOLE(t *testing.T) { map[string]expectedResult{ "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE", OutputSize: base.Byte(2)}}}, + "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, }, }, } From 19d56576b613a0bb6193361982da51cc68ced4d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Freddy=20Rom=C3=A1n?= Date: Sun, 1 May 2022 13:27:04 -0700 Subject: [PATCH 7/7] cr comments --- common/literalinput.go | 2 +- runner/runner.go | 7 ++++--- runner/runner_test.go | 22 ++++++++++++++-------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/common/literalinput.go b/common/literalinput.go index ee5f607..81388a3 100644 --- a/common/literalinput.go +++ b/common/literalinput.go @@ -143,7 +143,7 @@ var ( MemoryLimit: base.Gibibyte + base.Gibibyte/2, OverallWallTimeLimit: base.Duration(time.Duration(10) * time.Second), ExtraWallTime: base.Duration(0), - OutputLimit: base.Byte(100) * base.Mebibyte, + OutputLimit: base.Byte(20) * base.Mebibyte, } DefaultLiteralValidatorSettings = LiteralValidatorSettings{ diff --git a/runner/runner.go b/runner/runner.go index 633ed30..bca00a3 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1019,7 +1019,8 @@ func Grade( ) } runMeta = &RunMetadata{ - Verdict: "OK", + Verdict: "OK", + OutputSize: base.Byte(len(file.contents)), } if file.ole { runMeta.Verdict = "OLE" @@ -1199,8 +1200,8 @@ func Grade( totalWallTime, intermediateResult.runMeta.WallTime, ) - totalMemory = base.MaxBytes(totalMemory, intermediateResult.runMeta.Memory) - totalOutput = base.MaxBytes(totalOutput, intermediateResult.runMeta.OutputSize) + totalMemory += intermediateResult.runMeta.Memory + totalOutput += intermediateResult.runMeta.OutputSize } } close(metaChan) diff --git a/runner/runner_test.go b/runner/runner_test.go index a9b0f4e..cc82050 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -1294,8 +1294,8 @@ func TestGradeOLE(t *testing.T) { big.NewRat(1, 4), expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, map[string]expectedResult{ - "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, - "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, }, }, @@ -1307,8 +1307,8 @@ func TestGradeOLE(t *testing.T) { big.NewRat(1, 4), expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, map[string]expectedResult{ - "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, - "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, }, }, @@ -1323,8 +1323,8 @@ func TestGradeOLE(t *testing.T) { big.NewRat(1, 4), expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, map[string]expectedResult{ - "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, }, }, @@ -1344,8 +1344,8 @@ func TestGradeOLE(t *testing.T) { big.NewRat(1, 4), expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}}, map[string]expectedResult{ - "0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, - "1.0": {runOutput: programOutput{"3", "", &RunMetadata{Verdict: "OK"}}}, + "0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, + "1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}}, "1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}}, }, }, @@ -1384,6 +1384,12 @@ func TestGradeOLE(t *testing.T) { rte.expectedScore.String(), ) } + if results.OverallOutput != base.Byte(4) { + t.Errorf( + "results.OverallOutput = %d, expected 4", + results.OverallOutput.Bytes(), + ) + } }) } })