Skip to content

Commit

Permalink
Overall Output Limit (#79)
Browse files Browse the repository at this point in the history 
Intento de resolver #32. La idea es que se valga tener archivos grandes, nomás no muchos.

(De hecho esto es mejoría: creo que no había impedimento de, digamos, hacer un problema con 100 casos que corran súper rápido con output de 2MiB cada uno)
  • Loading branch information
@frcepeda
frcepeda authored Aug 29, 2022
1 parent 212b90b commit cb67a6f
Show file tree
Hide file tree
Showing 6 changed files with 234 additions and 18 deletions.
4 changes: 4 additions & 0 deletions common/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ type RunnerConfig struct {
RuntimePath string
CompileTimeLimit base.Duration
CompileOutputLimit base.Byte
HardMemoryLimit base.Byte
OverallOutputLimit base.Byte
OmegajailRoot string
PreserveFiles bool
}
Expand Down Expand Up @@ -202,6 +204,8 @@ var defaultConfig = Config{
GraderURL: "https://omegaup.com:11302",
CompileTimeLimit: base.Duration(time.Duration(30) * time.Second),
CompileOutputLimit: base.Byte(10) * base.Mebibyte,
HardMemoryLimit: base.Byte(640) * base.Mebibyte,
OverallOutputLimit: base.Byte(100) * base.Mebibyte,
OmegajailRoot: "/var/lib/omegajail",
PreserveFiles: false,
},
Expand Down
2 changes: 1 addition & 1 deletion common/literalinput.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ var (
MemoryLimit: base.Gibibyte + base.Gibibyte/2,
OverallWallTimeLimit: base.Duration(time.Duration(10) * time.Second),
ExtraWallTime: base.Duration(0),
OutputLimit: base.Byte(2) * base.Mebibyte,
OutputLimit: base.Byte(20) * base.Mebibyte,
}

DefaultLiteralValidatorSettings = LiteralValidatorSettings{
Expand Down
58 changes: 45 additions & 13 deletions runner/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,17 +142,18 @@ func (g *GroupResult) UnmarshalJSON(data []byte) error {

// A RunResult represents the results of a run.
type RunResult struct {
Verdict string `json:"verdict"`
CompileError *string `json:"compile_error,omitempty"`
CompileMeta map[string]RunMetadata `json:"compile_meta"`
Score *big.Rat `json:"score"`
ContestScore *big.Rat `json:"contest_score"`
MaxScore *big.Rat `json:"max_score"`
Time float64 `json:"time"`
WallTime float64 `json:"wall_time"`
Memory base.Byte `json:"memory"`
JudgedBy string `json:"judged_by,omitempty"`
Groups []GroupResult `json:"groups"`
Verdict string `json:"verdict"`
CompileError *string `json:"compile_error,omitempty"`
CompileMeta map[string]RunMetadata `json:"compile_meta"`
Score *big.Rat `json:"score"`
ContestScore *big.Rat `json:"contest_score"`
MaxScore *big.Rat `json:"max_score"`
Time float64 `json:"time"`
WallTime float64 `json:"wall_time"`
Memory base.Byte `json:"memory"`
OverallOutput base.Byte `json:"total_output"`
JudgedBy string `json:"judged_by,omitempty"`
Groups []GroupResult `json:"groups"`
}

// NewRunResult returns a new RunResult.
Expand Down Expand Up @@ -395,6 +396,7 @@ func parseOutputOnlyFile(
) (map[string]outputOnlyFile, error) {
dataURL, err := dataurl.DecodeString(data)
result := make(map[string]outputOnlyFile)
overallOutput := base.Byte(0)
if err != nil {
// |data| is not a dataurl. Try just returning the data as an Entry.
ctx.Log.Info(
Expand Down Expand Up @@ -460,6 +462,18 @@ func parseOutputOnlyFile(
result[fileName] = outputOnlyFile{"", true}
continue
}
if overallOutput > ctx.Config.Runner.OverallOutputLimit {
ctx.Log.Info(
"Output-only overall size limit has been exceeded. Generating empty file",
map[string]interface{}{
"name": f.FileHeader.Name,
"overall output": overallOutput,
"limit": ctx.Config.Runner.OverallOutputLimit,
},
)
result[fileName] = outputOnlyFile{"", true}
continue
}
rc, err := f.Open()
if err != nil {
ctx.Log.Info(
Expand All @@ -485,6 +499,7 @@ func parseOutputOnlyFile(
continue
}
result[fileName] = outputOnlyFile{buf.String(), false}
overallOutput += base.Byte(buf.Len())
}
return result, nil
}
Expand Down Expand Up @@ -974,6 +989,18 @@ func Grade(
runMeta = &RunMetadata{
Verdict: "TLE",
}
} else if runResult.OverallOutput > ctx.Config.Runner.OverallOutputLimit {
ctx.Log.Debug(
"Not even running since the overall output limit has been exceeded",
map[string]interface{}{
"case": caseData.Name,
"overall output": runResult.OverallOutput,
"limit": ctx.Config.Runner.OverallOutputLimit,
},
)
runMeta = &RunMetadata{
Verdict: "OLE",
}
} else if run.Language == "cat" {
outName := fmt.Sprintf("%s.out", caseData.Name)
errName := fmt.Sprintf("%s.err", caseData.Name)
Expand All @@ -992,7 +1019,8 @@ func Grade(
)
}
runMeta = &RunMetadata{
Verdict: "OK",
Verdict: "OK",
OutputSize: base.Byte(len(file.contents)),
}
if file.ole {
runMeta.Verdict = "OLE"
Expand Down Expand Up @@ -1146,6 +1174,7 @@ func Grade(
var totalTime float64
var totalWallTime float64
var totalMemory base.Byte
var totalOutput base.Byte
for i := 0; i < regularBinaryCount; i++ {
intermediateResult := <-metaChan
generatedFiles = append(generatedFiles, intermediateResult.generatedFiles...)
Expand All @@ -1171,7 +1200,8 @@ func Grade(
totalWallTime,
intermediateResult.runMeta.WallTime,
)
totalMemory += base.MaxBytes(totalMemory, intermediateResult.runMeta.Memory)
totalMemory += intermediateResult.runMeta.Memory
totalOutput += intermediateResult.runMeta.OutputSize
}
}
close(metaChan)
Expand All @@ -1180,13 +1210,15 @@ func Grade(
chosenMetadata.Time = totalTime
chosenMetadata.WallTime = totalWallTime
chosenMetadata.Memory = totalMemory
chosenMetadata.OutputSize = totalOutput

runMeta = mergeVerdict(ctx, &chosenMetadata, parentMetadata)
}
runResult.Verdict = worseVerdict(runResult.Verdict, runMeta.Verdict)
runResult.Time += runMeta.Time
runResult.WallTime += runMeta.WallTime
runResult.Memory = base.MaxBytes(runResult.Memory, runMeta.Memory)
runResult.OverallOutput += runMeta.OutputSize

// TODO: change CaseResult to split original metadatas and final metadata
caseResults[j] = CaseResult{
Expand Down
169 changes: 169 additions & 0 deletions runner/runner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1227,6 +1227,175 @@ func TestGradeLowMemOmegajail(t *testing.T) {
}
}

func TestGradeOLE(t *testing.T) {
for name, wrapper := range map[string]sandboxWrapper{
"fake": &fakeSandboxWrapper{},
"omegajail": &omegajailSandboxWrapper{omegajail: getSandbox()},
} {
wrapper := wrapper
t.Run(name, func(t *testing.T) {
if testing.Short() && wrapper.name() == "OmegajailSandbox" {
t.Skip("skipping test in short mode.")
}
if !wrapper.supported() {
t.Skip(fmt.Sprintf("%s not supported", wrapper.name()))
}

ctx, err := newRunnerContext(t)
if err != nil {
t.Fatalf("RunnerContext creation failed with %q", err)
}
defer ctx.Close()
if !ctx.Config.Runner.PreserveFiles {
defer os.RemoveAll(ctx.Config.Runner.RuntimePath)
}

// Artificially low limit for the test
ctx.Config.Runner.OverallOutputLimit = base.Byte(3)

inputManager := common.NewInputManager(ctx)
AplusB, err := common.NewLiteralInputFactory(
&common.LiteralInput{
Cases: map[string]*common.LiteralCaseSettings{
"0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)},
"1.0": {Input: "1 2", ExpectedOutput: "3", Weight: big.NewRat(1, 1)},
"1.1": {Input: "2 3", ExpectedOutput: "5", Weight: big.NewRat(2, 1)},
},
Validator: &common.LiteralValidatorSettings{
Name: common.ValidatorNameTokenNumeric,
},
Limits: &common.LimitsSettings{
TimeLimit: common.DefaultLiteralLimitSettings.TimeLimit,
MemoryLimit: 8 * 1024 * 1024,
OverallWallTimeLimit: common.DefaultLiteralLimitSettings.OverallWallTimeLimit,
ExtraWallTime: common.DefaultLiteralLimitSettings.ExtraWallTime,
OutputLimit: common.DefaultLiteralLimitSettings.OutputLimit,
},
},
ctx.Config.Runner.RuntimePath,
common.LiteralPersistRunner,
)
if err != nil {
t.Fatalf("Failed to create Input: %q", err)
}

inputRef, err := inputManager.Add(AplusB.Hash(), AplusB)
if err != nil {
t.Fatalf("Failed to open problem: %q", err)
}
defer inputRef.Release()

runtests := []runnerTestCase{
{
"c11-gcc",
"#include <stdio.h>\nint main() { printf(\"3\\n\"); }",
big.NewRat(1, 1),
"OLE",
big.NewRat(1, 4),
expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}},
map[string]expectedResult{
"0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}},
},
},
{
"cpp17-gcc",
"#include <iostream>\nint main() { std::cout << \"3\\n\"; }",
big.NewRat(1, 1),
"OLE",
big.NewRat(1, 4),
expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}},
map[string]expectedResult{
"0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}},
},
},
{
"pas",
`program Main;
begin
writeln ('3');
end.`,
big.NewRat(1, 1),
"OLE",
big.NewRat(1, 4),
expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}},
map[string]expectedResult{
"0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}},
},
},
{
"cat",
"data:application/zip;base64,UEsDBAoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABwAMC5vdX" +
"RVVAkAA67WxFb8t4ZYdXgLAAEE6AMAAAToAwAAMwpQSwMECgAAAAAAhhE4StGeZ1UCAAAAAg" +
"AAAAcAHAAxLjAub3V0VVQJAAP8t4ZYCbiGWHV4CwABBOgDAAAE6AMAADMKUEsDBAoAAAAAAO" +
"eiUUhXOT0DAgAAAAIAAAAHABwAMS4xLm91dFVUCQADstbEVgm4hlh1eAsAAQToAwAABOgDAA" +
"A1ClBLAQIeAwoAAAAAAOWiUUjRnmdVAgAAAAIAAAAFABgAAAAAAAEAAAC0gQAAAAAwLm91dF" +
"VUBQADrtbEVnV4CwABBOgDAAAE6AMAAFBLAQIeAwoAAAAAAIYROErRnmdVAgAAAAIAAAAHAB" +
"gAAAAAAAEAAAC0gUEAAAAxLjAub3V0VVQFAAP8t4ZYdXgLAAEE6AMAAAToAwAAUEsBAh4DCg" +
"AAAAAA56JRSFc5PQMCAAAAAgAAAAcAGAAAAAAAAQAAALSBhAAAADEuMS5vdXRVVAUAA7LWxF" +
"Z1eAsAAQToAwAABOgDAABQSwUGAAAAAAMAAwDlAAAAxwAAAAAA",
big.NewRat(1, 1),
"OLE",
big.NewRat(1, 4),
expectedResult{runOutput: programOutput{"", "", &RunMetadata{Verdict: "OK"}}},
map[string]expectedResult{
"0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.0": {runOutput: programOutput{"3\n", "", &RunMetadata{Verdict: "OK", OutputSize: base.Byte(2)}}},
"1.1": {runOutput: programOutput{"", "", &RunMetadata{Verdict: "OLE"}}},
},
},
}
for idx, rte := range runtests {
t.Run(fmt.Sprintf("%s/%d/%s %s", wrapper.name(), idx, rte.language, rte.expectedVerdict), func(t *testing.T) {
results, err := Grade(
ctx,
&bytes.Buffer{},
&common.Run{
AttemptID: uint64(idx),
Language: rte.language,
InputHash: inputRef.Input.Hash(),
Source: rte.source,
MaxScore: rte.maxScore,
},
inputRef.Input,
wrapper.sandbox(&rte),
)
if err != nil {
t.Fatalf("Failed to run %v: %q", rte, err)
}
if results.Verdict != rte.expectedVerdict {
t.Errorf(
"results.Verdict = %q, expected %q, test %v: %v",
results.Verdict,
rte.expectedVerdict,
idx,
rte,
)
}
if results.Score.Cmp(rte.expectedScore) != 0 {
t.Errorf(
"results.Score = %s, expected %s",
results.Score.String(),
rte.expectedScore.String(),
)
}
if results.OverallOutput != base.Byte(4) {
t.Errorf(
"results.OverallOutput = %d, expected 4",
results.OverallOutput.Bytes(),
)
}
})
}
})
}
}

func TestKarelGrade(t *testing.T) {
for name, wrapper := range map[string]sandboxWrapper{
"fake": &fakeSandboxWrapper{},
Expand Down
Loading

0 comments on commit cb67a6f

Please sign in to comment.