Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Overall Output Limit #79

Merged
merged 8 commits into from
Aug 29, 2022
Merged

Overall Output Limit #79

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4f12b9b
rfc
frcepeda Apr 30, 2022
2e78224
OLE test
frcepeda May 1, 2022
fc58b70
better tests
frcepeda May 1, 2022
efe4c53
even better tests
frcepeda May 1, 2022
4a71b7f
?
frcepeda May 1, 2022
ca0e5ac
counting is hard
frcepeda May 1, 2022
19d5657
cr comments
frcepeda May 1, 2022
6842ede
Merge branch 'main' into cumulative-output-limit
frcepeda Jul 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions common/context.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ type RunnerConfig struct {
RuntimePath string
CompileTimeLimit base.Duration
CompileOutputLimit base.Byte
HardMemoryLimit base.Byte
OverallOutputLimit base.Byte
OmegajailRoot string
PreserveFiles bool
}
Expand Down Expand Up @@ -202,6 +204,8 @@ var defaultConfig = Config{
GraderURL: "https://omegaup.com:11302",
CompileTimeLimit: base.Duration(time.Duration(30) * time.Second),
CompileOutputLimit: base.Byte(10) * base.Mebibyte,
HardMemoryLimit: base.Byte(640) * base.Mebibyte,
OverallOutputLimit: base.Byte(100) * base.Mebibyte,
OmegajailRoot: "/var/lib/omegajail",
PreserveFiles: false,
},
Expand Down
2 changes: 1 addition & 1 deletion common/literalinput.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ var (
MemoryLimit: base.Gibibyte + base.Gibibyte/2,
OverallWallTimeLimit: base.Duration(time.Duration(10) * time.Second),
ExtraWallTime: base.Duration(0),
OutputLimit: base.Byte(2) * base.Mebibyte,
OutputLimit: base.Byte(20) * base.Mebibyte,
Copy link
Member Author

@frcepeda frcepeda May 1, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lo dejé en 20—la cosa es que luego sí hay problemas con casos grandotes que omegaUp no ha podido soportar por esto. (O supongo 10 está bien pero reevaluar si seguimos topándonos con este límite; 2 MiB definitivamente era muy poquito)

}

DefaultLiteralValidatorSettings = LiteralValidatorSettings{
Expand Down
58 changes: 45 additions & 13 deletions runner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,17 +142,18 @@ func (g *GroupResult) UnmarshalJSON(data []byte) error {

// A RunResult represents the results of a run.
type RunResult struct {
Verdict string `json:"verdict"`
CompileError *string `json:"compile_error,omitempty"`
CompileMeta map[string]RunMetadata `json:"compile_meta"`
Score *big.Rat `json:"score"`
ContestScore *big.Rat `json:"contest_score"`
MaxScore *big.Rat `json:"max_score"`
Time float64 `json:"time"`
WallTime float64 `json:"wall_time"`
Memory base.Byte `json:"memory"`
JudgedBy string `json:"judged_by,omitempty"`
Groups []GroupResult `json:"groups"`
Verdict string `json:"verdict"`
CompileError *string `json:"compile_error,omitempty"`
CompileMeta map[string]RunMetadata `json:"compile_meta"`
Score *big.Rat `json:"score"`
ContestScore *big.Rat `json:"contest_score"`
MaxScore *big.Rat `json:"max_score"`
Time float64 `json:"time"`
WallTime float64 `json:"wall_time"`
Memory base.Byte `json:"memory"`
OverallOutput base.Byte `json:"total_output"`
JudgedBy string `json:"judged_by,omitempty"`
Groups []GroupResult `json:"groups"`
}

// NewRunResult returns a new RunResult.
Expand Down Expand Up @@ -395,6 +396,7 @@ func parseOutputOnlyFile(
) (map[string]outputOnlyFile, error) {
dataURL, err := dataurl.DecodeString(data)
result := make(map[string]outputOnlyFile)
overallOutput := base.Byte(0)
if err != nil {
// |data| is not a dataurl. Try just returning the data as an Entry.
ctx.Log.Info(
Expand Down Expand Up @@ -460,6 +462,18 @@ func parseOutputOnlyFile(
result[fileName] = outputOnlyFile{"", true}
continue
}
if overallOutput > ctx.Config.Runner.OverallOutputLimit {
ctx.Log.Info(
"Output-only overall size limit has been exceeded. Generating empty file",
map[string]interface{}{
"name": f.FileHeader.Name,
"overall output": overallOutput,
"limit": ctx.Config.Runner.OverallOutputLimit,
},
)
result[fileName] = outputOnlyFile{"", true}
continue
}
rc, err := f.Open()
if err != nil {
ctx.Log.Info(
Expand All @@ -485,6 +499,7 @@ func parseOutputOnlyFile(
continue
}
result[fileName] = outputOnlyFile{buf.String(), false}
overallOutput += base.Byte(buf.Len())
}
return result, nil
}
Expand Down Expand Up @@ -974,6 +989,18 @@ func Grade(
runMeta = &RunMetadata{
Verdict: "TLE",
}
} else if runResult.OverallOutput > ctx.Config.Runner.OverallOutputLimit {
ctx.Log.Debug(
"Not even running since the overall output limit has been exceeded",
map[string]interface{}{
"case": caseData.Name,
"overall output": runResult.OverallOutput,
"limit": ctx.Config.Runner.OverallOutputLimit,
},
)
runMeta = &RunMetadata{
Verdict: "OLE",
}
} else if run.Language == "cat" {
outName := fmt.Sprintf("%s.out", caseData.Name)
errName := fmt.Sprintf("%s.err", caseData.Name)
Expand All @@ -992,7 +1019,8 @@ func Grade(
)
}
runMeta = &RunMetadata{
Verdict: "OK",
Verdict: "OK",
OutputSize: base.Byte(len(file.contents)),
}
if file.ole {
runMeta.Verdict = "OLE"
Expand Down Expand Up @@ -1146,6 +1174,7 @@ func Grade(
var totalTime float64
var totalWallTime float64
var totalMemory base.Byte
var totalOutput base.Byte
for i := 0; i < regularBinaryCount; i++ {
intermediateResult := <-metaChan
generatedFiles = append(generatedFiles, intermediateResult.generatedFiles...)
Expand All @@ -1171,7 +1200,8 @@ func Grade(
totalWallTime,
intermediateResult.runMeta.WallTime,
)
totalMemory += base.MaxBytes(totalMemory, intermediateResult.runMeta.Memory)
frcepeda marked this conversation as resolved.
Show resolved Hide resolved
totalMemory += intermediateResult.runMeta.Memory
totalOutput += intermediateResult.runMeta.OutputSize
}
}
close(metaChan)
Expand All @@ -1180,13 +1210,15 @@ func Grade(
chosenMetadata.Time = totalTime
chosenMetadata.WallTime = totalWallTime
chosenMetadata.Memory = totalMemory
chosenMetadata.OutputSize = totalOutput

runMeta = mergeVerdict(ctx, &chosenMetadata, parentMetadata)
}
runResult.Verdict = worseVerdict(runResult.Verdict, runMeta.Verdict)
runResult.Time += runMeta.Time
runResult.WallTime += runMeta.WallTime
runResult.Memory = base.MaxBytes(runResult.Memory, runMeta.Memory)
runResult.OverallOutput += runMeta.OutputSize

// TODO: change CaseResult to split original metadatas and final metadata
caseResults[j] = CaseResult{
Expand Down
169 changes: 169 additions & 0 deletions runner/runner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1227,6 +1227,175 @@ func TestGradeLowMemOmegajail(t *testing.T) {
}
}

func TestGradeOLE(t *testing.T) {
for name, wrapper := range map[string]sandboxWrapper{
"fake": &fakeSandboxWrapper{},
"omegajail": &omegajailSandboxWrapper{omegajail: getSandbox()},
} {
wrapper := wrapper
t.Run(name, func(t *testing.T) {
if testing.Short() && wrapper.name() == "OmegajailSandbox" {
t.Skip("skipping test in short mode.")
}
if !wrapper.supported() {
t.Skip(fmt.Sprintf("%s not supported", wrapper.name()))
}

ctx, err := newRunnerContext(t)
if err != nil {
t.Fatalf("RunnerContext creation failed with %q", err)
}
defer ctx.Close()
if !ctx.Config.Runner.PreserveFiles {
defer os.RemoveAll(ctx.Config.Runner.RuntimePath)
}

// Artificially low limit for the test
ctx.Config.Runner.OverallOutputLimit = base.Byte(3)

inputManager := common.NewInputManager(ctx)
AplusB, err := common.NewLiteralInputFactory(
&common.LiteralInput{
Cases: map[string]*common.LiteralCaseSettings{
"0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)},
"1.0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)},
"1.1": {Input: "2 3", ExpectedOutput: "5", Weight: big.NewRat(2, 1)},
},
Validator: &common.LiteralValidatorSettings{
Name: common.ValidatorNameTokenNumeric,
},
Limits: &common.LimitsSettings{
TimeLimit: common.DefaultLiteralLimitSettings.TimeLimit,
MemoryLimit: 8 * 1024 * 1024,
OverallWallTimeLimit: common.DefaultLiteralLimitSettings.OverallWallTimeLimit,
ExtraWallTime: common.DefaultLiteralLimitSettings.ExtraWallTime,
OutputLimit: common.DefaultLiteralLimitSettings.OutputLimit,
},
},
ctx.Config.Runner.RuntimePath,
common.LiteralPersistRunner,
)
if err != nil {
t.Fatalf("Failed to create Input: %q", err)
}

inputRef, err := inputManager.Add(AplusB.Hash(), AplusB)
if err != nil {
t.Fatalf("Failed to open problem: %q", err)
}
defer inputRef.Release()

runtests := []runnerTestCase{
{
"c11-gcc",
"#include <stdio.h>\nint main() { printf(\"3\\n\"); }",
big.NewRat(1, 1),
"OLE",
big.NewRat(1, 4),
expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}},
map[string]expectedResult{
"0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}},
},
},
{
"cpp17-gcc",
"#include <iostream>\nint main() { std::cout << \"3\\n\"; }",
big.NewRat(1, 1),
"OLE",
big.NewRat(1, 4),
expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}},
map[string]expectedResult{
"0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}},
},
},
{
"pas",
`program Main;
begin
writeln ('3');
end.`,
big.NewRat(1, 1),
"OLE",
big.NewRat(1, 4),
expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}},
map[string]expectedResult{
"0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}},
},
},
{
"cat",
"data:application/zip;base64,UEsDBAoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABwAMC5vdX" +
"RVVAkAA67WxFb8t4ZYdXgLAAEE6AMAAAToAwAAMwpQSwMECgAAAAAAhhE4StGeZ1UCAAAAAg" +
"AAAAcAHAAxLjAub3V0VVQJAAP8t4ZYCbiGWHV4CwABBOgDAAAE6AMAADMKUEsDBAoAAAAAAO" +
"eiUUhXOT0DAgAAAAIAAAAHABwAMS4xLm91dFVUCQADstbEVgm4hlh1eAsAAQToAwAABOgDAA" +
"A1ClBLAQIeAwoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABgAAAAAAAEAAAC0gQAAAAAwLm91dF" +
"VUBQADrtbEVnV4CwABBOgDAAAE6AMAAFBLAQIeAwoAAAAAAIYROErRnmdVAgAAAAIAAAAHAB" +
"gAAAAAAAEAAAC0gUEAAAAxLjAub3V0VVQFAAP8t4ZYdXgLAAEE6AMAAAToAwAAUEsBAh4DCg" +
"AAAAAA56JRSFc5PQMCAAAAAgAAAAcAGAAAAAAAAQAAALSBhAAAADEuMS5vdXRVVAUAA7LWxF" +
"Z1eAsAAQToAwAABOgDAABQSwUGAAAAAAMAAwDlAAAAxwAAAAAA",
big.NewRat(1, 1),
"OLE",
big.NewRat(1, 4),
expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}},
map[string]expectedResult{
"0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}},
},
},
}
for idx, rte := range runtests {
t.Run(fmt.Sprintf("%s/%d/%s %s", wrapper.name(), idx, rte.language, rte.expectedVerdict), func(t *testing.T) {
frcepeda marked this conversation as resolved.
Show resolved Hide resolved
results, err := Grade(
ctx,
&bytes.Buffer{},
&common.Run{
AttemptID: uint64(idx),
Language: rte.language,
InputHash: inputRef.Input.Hash(),
Source: rte.source,
MaxScore: rte.maxScore,
},
inputRef.Input,
wrapper.sandbox(&rte),
)
if err != nil {
t.Fatalf("Failed to run %v: %q", rte, err)
}
if results.Verdict != rte.expectedVerdict {
t.Errorf(
"results.Verdict = %q, expected %q, test %v: %v",
results.Verdict,
rte.expectedVerdict,
idx,
rte,
)
}
if results.Score.Cmp(rte.expectedScore) != 0 {
t.Errorf(
"results.Score = %s, expected %s",
results.Score.String(),
rte.expectedScore.String(),
)
}
if results.OverallOutput != base.Byte(4) {
t.Errorf(
"results.OverallOutput = %d, expected 4",
results.OverallOutput.Bytes(),
)
}
})
}
})
}
}

func TestKarelGrade(t *testing.T) {
for name, wrapper := range map[string]sandboxWrapper{
"fake": &fakeSandboxWrapper{},
Expand Down
Loading