Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Overall Output Limit #79

Merged
merged 8 commits into from
Aug 29, 2022
Merged

Overall Output Limit #79

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4f12b9b
rfc
frcepeda Apr 30, 2022
2e78224
OLE test
frcepeda May 1, 2022
fc58b70
better tests
frcepeda May 1, 2022
efe4c53
even better tests
frcepeda May 1, 2022
4a71b7f
?
frcepeda May 1, 2022
ca0e5ac
counting is hard
frcepeda May 1, 2022
19d5657
cr comments
frcepeda May 1, 2022
6842ede
Merge branch 'main' into cumulative-output-limit
frcepeda Jul 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion common/literalinput.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,8 @@ var (
MemoryLimit: base.Gibibyte + base.Gibibyte/2,
OverallWallTimeLimit: base.Duration(time.Duration(10) * time.Second),
ExtraWallTime: base.Duration(0),
OutputLimit: base.Byte(2) * base.Mebibyte,
OutputLimit: base.Byte(100) * base.Mebibyte,
frcepeda marked this conversation as resolved.
Show resolved Hide resolved
OverallOutputLimit: base.Byte(100) * base.Mebibyte,
}

DefaultLiteralValidatorSettings = LiteralValidatorSettings{
Expand Down Expand Up @@ -306,6 +307,10 @@ func NewLiteralInputFactory(
input.Limits.OutputLimit,
DefaultLiteralLimitSettings.OutputLimit,
)
settings.Limits.OverallOutputLimit = base.MinBytes(
input.Limits.OverallOutputLimit,
DefaultLiteralLimitSettings.OverallOutputLimit,
)
} else {
settings.Limits = DefaultLiteralLimitSettings
}
Expand Down
3 changes: 3 additions & 0 deletions common/problemsettings.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ type LimitsSettings struct {
ExtraWallTime base.Duration
MemoryLimit base.Byte
OutputLimit base.Byte
OverallOutputLimit base.Byte
OverallWallTimeLimit base.Duration
TimeLimit base.Duration
}
Expand Down Expand Up @@ -189,6 +190,7 @@ var (
OutputLimit: base.Byte(10) * base.Kibibyte,
OverallWallTimeLimit: base.Duration(time.Duration(5) * time.Second),
TimeLimit: base.Duration(time.Duration(1) * time.Second),
OverallOutputLimit: base.Byte(100) * base.Mebibyte,
}

// DefaultLimits specifies the default limits for a problem.
Expand All @@ -198,6 +200,7 @@ var (
OutputLimit: base.Byte(10) * base.Kibibyte,
OverallWallTimeLimit: base.Duration(time.Duration(1) * time.Minute),
TimeLimit: base.Duration(time.Duration(1) * time.Second),
OverallOutputLimit: base.Byte(100) * base.Mebibyte,
}
)

Expand Down
55 changes: 43 additions & 12 deletions runner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,17 +142,18 @@ func (g *GroupResult) UnmarshalJSON(data []byte) error {

// A RunResult represents the results of a run.
type RunResult struct {
Verdict string `json:"verdict"`
CompileError *string `json:"compile_error,omitempty"`
CompileMeta map[string]RunMetadata `json:"compile_meta"`
Score *big.Rat `json:"score"`
ContestScore *big.Rat `json:"contest_score"`
MaxScore *big.Rat `json:"max_score"`
Time float64 `json:"time"`
WallTime float64 `json:"wall_time"`
Memory base.Byte `json:"memory"`
JudgedBy string `json:"judged_by,omitempty"`
Groups []GroupResult `json:"groups"`
Verdict string `json:"verdict"`
CompileError *string `json:"compile_error,omitempty"`
CompileMeta map[string]RunMetadata `json:"compile_meta"`
Score *big.Rat `json:"score"`
ContestScore *big.Rat `json:"contest_score"`
MaxScore *big.Rat `json:"max_score"`
Time float64 `json:"time"`
WallTime float64 `json:"wall_time"`
Memory base.Byte `json:"memory"`
OverallOutput base.Byte `json:"total_output"`
JudgedBy string `json:"judged_by,omitempty"`
Groups []GroupResult `json:"groups"`
}

// NewRunResult returns a new RunResult.
Expand Down Expand Up @@ -395,6 +396,7 @@ func parseOutputOnlyFile(
) (map[string]outputOnlyFile, error) {
dataURL, err := dataurl.DecodeString(data)
result := make(map[string]outputOnlyFile)
overallOutput := base.Byte(0)
if err != nil {
// |data| is not a dataurl. Try just returning the data as an Entry.
ctx.Log.Info(
Expand Down Expand Up @@ -460,6 +462,18 @@ func parseOutputOnlyFile(
result[fileName] = outputOnlyFile{"", true}
continue
}
if overallOutput > settings.Limits.OverallOutputLimit {
ctx.Log.Info(
"Output-only overall size limit has been exceeded. Generating empty file",
map[string]interface{}{
"name": f.FileHeader.Name,
"overall output": overallOutput,
"limit": settings.Limits.OverallOutputLimit,
},
)
result[fileName] = outputOnlyFile{"", true}
continue
}
rc, err := f.Open()
if err != nil {
ctx.Log.Info(
Expand All @@ -485,6 +499,7 @@ func parseOutputOnlyFile(
continue
}
result[fileName] = outputOnlyFile{buf.String(), false}
overallOutput += base.Byte(buf.Len())
}
return result, nil
}
Expand Down Expand Up @@ -974,6 +989,18 @@ func Grade(
runMeta = &RunMetadata{
Verdict: "TLE",
}
} else if runResult.OverallOutput > settings.Limits.OverallOutputLimit {
ctx.Log.Debug(
"Not even running since the overall output limit has been exceeded",
map[string]interface{}{
"case": caseData.Name,
"overall output": runResult.OverallOutput,
"limit": settings.Limits.OverallOutputLimit,
},
)
runMeta = &RunMetadata{
Verdict: "OLE",
}
} else if run.Language == "cat" {
outName := fmt.Sprintf("%s.out", caseData.Name)
errName := fmt.Sprintf("%s.err", caseData.Name)
Expand Down Expand Up @@ -1146,6 +1173,7 @@ func Grade(
var totalTime float64
var totalWallTime float64
var totalMemory base.Byte
var totalOutput base.Byte
for i := 0; i < regularBinaryCount; i++ {
intermediateResult := <-metaChan
generatedFiles = append(generatedFiles, intermediateResult.generatedFiles...)
Expand All @@ -1171,7 +1199,8 @@ func Grade(
totalWallTime,
intermediateResult.runMeta.WallTime,
)
totalMemory += base.MaxBytes(totalMemory, intermediateResult.runMeta.Memory)
frcepeda marked this conversation as resolved.
Show resolved Hide resolved
totalMemory = base.MaxBytes(totalMemory, intermediateResult.runMeta.Memory)
totalOutput = base.MaxBytes(totalOutput, intermediateResult.runMeta.OutputSize)
frcepeda marked this conversation as resolved.
Show resolved Hide resolved
}
}
close(metaChan)
Expand All @@ -1180,13 +1209,15 @@ func Grade(
chosenMetadata.Time = totalTime
chosenMetadata.WallTime = totalWallTime
chosenMetadata.Memory = totalMemory
chosenMetadata.OutputSize = totalOutput

runMeta = mergeVerdict(ctx, &chosenMetadata, parentMetadata)
}
runResult.Verdict = worseVerdict(runResult.Verdict, runMeta.Verdict)
runResult.Time += runMeta.Time
runResult.WallTime += runMeta.WallTime
runResult.Memory = base.MaxBytes(runResult.Memory, runMeta.Memory)
runResult.OverallOutput += runMeta.OutputSize

// TODO: change CaseResult to split original metadatas and final metadata
caseResults[j] = CaseResult{
Expand Down
16 changes: 13 additions & 3 deletions runner/sandbox.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,19 +79,21 @@ type RunMetadata struct {
SystemTime float64 `json:"sys_time"`
WallTime float64 `json:"wall_time"`
Memory base.Byte `json:"memory"`
OutputSize base.Byte `json:"output_size"`
Signal *string `json:"signal,omitempty"`
Syscall *string `json:"syscall,omitempty"`
}

func (m *RunMetadata) String() string {
metadata := fmt.Sprintf(
"{Verdict: %s, ExitStatus: %d, Time: %.3fs, SystemTime: %.3fs, WallTime: %.3fs, Memory: %.3fMiB",
"{Verdict: %s, ExitStatus: %d, Time: %.3fs, SystemTime: %.3fs, WallTime: %.3fs, Memory: %.3fMiB, OutputSize: %.3fMiB",
m.Verdict,
m.ExitStatus,
m.Time,
m.SystemTime,
m.WallTime,
m.Memory.Mebibytes(),
m.OutputSize.Mebibytes(),
)
if m.Signal != nil {
metadata += fmt.Sprintf(", Signal: %s", *m.Signal)
Expand Down Expand Up @@ -220,7 +222,7 @@ func (o *OmegajailSandbox) Compile(
}, err
}
defer metaFd.Close()
metadata, err := parseMetaFile(ctx, nil, lang, metaFd, nil, false)
metadata, err := parseMetaFile(ctx, nil, lang, metaFd, &outputFile, nil, false)

if lang == "java" && metadata.Verdict == "OK" {
classPath := path.Join(chdir, fmt.Sprintf("%s.class", target))
Expand Down Expand Up @@ -358,7 +360,7 @@ func (o *OmegajailSandbox) Run(
}, err
}
defer metaFd.Close()
return parseMetaFile(ctx, limits, lang, metaFd, &errorFile, lang == "c")
return parseMetaFile(ctx, limits, lang, metaFd, &outputFile, &errorFile, lang == "c")
}

func invokeOmegajail(ctx *common.Context, omegajailRoot string, omegajailParams []string, errorFile string) {
Expand Down Expand Up @@ -433,6 +435,7 @@ func parseMetaFile(
limits *common.LimitsSettings,
lang string,
metaFile io.Reader,
outputFilePath *string,
errorFilePath *string,
allowNonZeroExitCode bool,
) (*RunMetadata, error) {
Expand Down Expand Up @@ -524,6 +527,13 @@ func parseMetaFile(
meta.Memory = limits.MemoryLimit
}

if outputFilePath != nil {
outputFileStat, err := os.Stat(*outputFilePath)
if err == nil {
meta.OutputSize = base.Byte(outputFileStat.Size())
}
}

return meta, nil
}

Expand Down
1 change: 1 addition & 0 deletions runner/sandbox_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ func TestParseMetaFile(t *testing.T) {
te.lang,
bytes.NewBufferString(te.contents),
nil,
nil,
te.lang == "c",
)
if err != nil {
Expand Down