Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

split summary.csv into summary and summary_extended #890

Merged
merged 10 commits into from
Nov 6, 2024
76 changes: 51 additions & 25 deletions src/main/java/network/brightspots/rcv/ResultsWriter.java
Original file line number Diff line number Diff line change
Expand Up @@ -280,26 +280,48 @@ void generateBySliceSummaryFiles(
RoundTallies roundTallies = entry.getValue();
TallyTransfers tallyTransfers = tallyTransfersBySlice.get(slice, sliceId);
String sliceFileString = getFileStringForSlice(slice, sliceId, filenames);
String outputPath = getOutputFilePathFromInstance(
String.format("%s_summary", sliceFileString));
generateSummarySpreadsheet(roundTallies, candidateOrder, slice, sliceId, outputPath);
generateSummaryJson(roundTallies, tallyTransfers, slice, sliceId, outputPath);
String outputPathCsv = getOutputFilePathFromInstance(
String.format("%s_extended_summary", sliceFileString));
artoonie marked this conversation as resolved.
Show resolved Hide resolved
String outputPathJson = getOutputFilePathFromInstance(
String.format("%s_summary", sliceFileString));
generateSummaryExtendedCsv(roundTallies, candidateOrder, slice, sliceId, outputPathCsv);
artoonie marked this conversation as resolved.
Show resolved Hide resolved
generateSummaryJson(roundTallies, tallyTransfers, slice, sliceId, outputPathJson);
}
}
}

private void generateSummaryExtendedCsv(
RoundTallies roundTallies,
List<String> candidateOrder,
TabulateBySlice slice,
String sliceId,
String outputPath) throws IOException {
generateSummaryCsvHelper(roundTallies, candidateOrder, slice, sliceId, outputPath, true);
}

private void generateSummaryCsv(
RoundTallies roundTallies,
List<String> candidateOrder,
TabulateBySlice slice,
String sliceId,
String outputPath) throws IOException {
generateSummaryCsvHelper(roundTallies, candidateOrder, slice, sliceId, outputPath, false);
}

// create a summary spreadsheet .csv file
// param: roundTallies is the round-by-count count of votes per candidate
// param: candidateOrder is to allow a consistent ordering of candidates, including across slices
// param: slice indicates which type of slice we're reporting results for (null means all)
// param: sliceId indicates the specific slice ID we're reporting results for (null means all)
// param: outputPath is the path to the output file, minus its extension
private void generateSummarySpreadsheet(
// param: extended include additional details in the output file?
private void generateSummaryCsvHelper(
RoundTallies roundTallies,
List<String> candidateOrder,
TabulateBySlice slice,
String sliceId,
String outputPath) throws IOException {
String outputPath,
boolean extended) throws IOException {
// Check that all candidates are included in the candidate order
Set<String> expectedCandidates = roundTallies.get(1).getCandidates();
Set<String> providedCandidates = new HashSet<>(candidateOrder);
Expand Down Expand Up @@ -363,7 +385,7 @@ private void generateSummarySpreadsheet(
} else {
votePctDivisor = roundTallies.get(1).activeAndLockedInBallotSum();
}
if (votePctDivisor != BigDecimal.ZERO) {
if (!votePctDivisor.equals(BigDecimal.ZERO)) {
// Turn a decimal into a human-readable percentage (e.g. 0.1234 -> 12.34%)
BigDecimal divDecimal = thisRoundTally.divide(votePctDivisor, MathContext.DECIMAL32);
csvPrinter.print(divDecimal.scaleByPowerOfTen(4).intValue() / 100.0 + "%");
Expand Down Expand Up @@ -405,26 +427,28 @@ private void generateSummarySpreadsheet(
csvPrinter.println();
}

for (StatusForRound status : STATUSES_TO_PRINT) {
csvPrinter.print(status.getTitleCaseKey());
if (extended) {
for (StatusForRound status : STATUSES_TO_PRINT) {
csvPrinter.print(status.getTitleCaseKey());

for (int round = 1; round <= numRounds; round++) {
BigDecimal thisRoundInactive = roundTallies.get(round).getBallotStatusTally(status);
csvPrinter.print(thisRoundInactive);
for (int round = 1; round <= numRounds; round++) {
BigDecimal thisRoundInactive = roundTallies.get(round).getBallotStatusTally(status);
csvPrinter.print(thisRoundInactive);

// Don't display percentage of inactive ballots
csvPrinter.print("");
// Don't display percentage of inactive ballots
csvPrinter.print("");

// Do display transfer of inactive ballots
if (round != numRounds) {
BigDecimal nextRoundInactive = roundTallies.get(round + 1).getBallotStatusTally(status);
BigDecimal diff = nextRoundInactive.subtract(thisRoundInactive);
csvPrinter.print(diff);
} else {
csvPrinter.print(0);
// Do display transfer of inactive ballots
if (round != numRounds) {
BigDecimal nextRoundInactive = roundTallies.get(round + 1).getBallotStatusTally(status);
BigDecimal diff = nextRoundInactive.subtract(thisRoundInactive);
csvPrinter.print(diff);
} else {
csvPrinter.print(0);
}
}
csvPrinter.println();
}
csvPrinter.println();
}

csvPrinter.print("Inactive Ballots Total");
Expand Down Expand Up @@ -585,9 +609,11 @@ void generateOverallSummaryFiles(
RoundTallies roundTallies,
TallyTransfers tallyTransfers,
List<String> candidateOrder) throws IOException {
String outputPath = getOutputFilePathFromInstance("summary");
generateSummarySpreadsheet(roundTallies, candidateOrder, null, null, outputPath);
generateSummaryJson(roundTallies, tallyTransfers, null, null, outputPath);
String outputPathSummary = getOutputFilePathFromInstance("summary");
String outputPathExtended = getOutputFilePathFromInstance("extended_summary");
generateSummaryExtendedCsv(roundTallies, candidateOrder, null, null, outputPathExtended);
generateSummaryCsv(roundTallies, candidateOrder, null, null, outputPathSummary);
generateSummaryJson(roundTallies, tallyTransfers, null, null, outputPathSummary);
}

// Write CastVoteRecords for the specified contest to the provided folder,
Expand Down
67 changes: 60 additions & 7 deletions src/test/java/network/brightspots/rcv/TabulatorTests.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@

package network.brightspots.rcv;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;

Expand All @@ -31,7 +33,6 @@
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
Expand Down Expand Up @@ -147,8 +148,8 @@ private static boolean compareLists(

private static boolean fileCompareLineByLine(String path1, String path2) {
boolean result = true;
try (BufferedReader br1 = new BufferedReader(new FileReader(path1, StandardCharsets.UTF_8));
BufferedReader br2 = new BufferedReader(new FileReader(path2, StandardCharsets.UTF_8))) {
try (BufferedReader br1 = new BufferedReader(new FileReader(path1, UTF_8));
BufferedReader br2 = new BufferedReader(new FileReader(path2, UTF_8))) {
int currentLine = 1;
int errorCount = 0;

Expand Down Expand Up @@ -232,12 +233,14 @@ private static void runTabulationTest(String stem, String expectedException,
int numSlicedFilesChecked = 0;
for (ContestConfig.TabulateBySlice slice : config.enabledSlices()) {
for (String sliceName : session.loadSliceNamesFromCvrs(slice, config)) {
String outputType = ResultsWriter.sanitizeStringForOutput(
String outputTypeJson = ResultsWriter.sanitizeStringForOutput(
String.format("%s_%s_summary", sliceName, slice.toLowerString()));
if (compareFiles(config, stem, outputType, ".json", timestampString, null, true)) {
String outputTypeCsv = ResultsWriter.sanitizeStringForOutput(
String.format("%s_%s_extended_summary", sliceName, slice.toLowerString()));
if (compareFiles(config, stem, outputTypeJson, ".json", timestampString, null, true)) {
numSlicedFilesChecked++;
}
if (compareFiles(config, stem, outputType, ".csv", timestampString, null, true)) {
if (compareFiles(config, stem, outputTypeCsv, ".csv", timestampString, null, true)) {
numSlicedFilesChecked++;
}
}
Expand Down Expand Up @@ -321,7 +324,8 @@ private static void cleanOutputFolder(TabulatorSession session) {
private static void compareFiles(
ContestConfig config, String stem, String timestampString, String sequentialId) {
compareFiles(config, stem, "summary", ".json", timestampString, sequentialId, false);
compareFiles(config, stem, "summary", ".csv", timestampString, sequentialId, false);
compareFiles(config, stem, "extended_summary", ".csv", timestampString, sequentialId, false);
compareExtendedSummaryToSummary(config, timestampString, sequentialId);
if (config.isGenerateCdfJsonEnabled()) {
compareFiles(config, stem, "cvr_cdf", ".json", timestampString, sequentialId, false);
}
Expand Down Expand Up @@ -365,6 +369,55 @@ private static boolean compareFiles(
return didCompare;
}

/**
* Rather than storing both the extended summary and non-extended summary files in git, we can
* directly check that the non-extended file is precisely what we expect: everything in the
* extended file except for the inactive ballot breakdown.
*/
private static void compareExtendedSummaryToSummary(
ContestConfig config, String timestampString, String sequentialId) {
String summaryPath = ResultsWriter.getOutputFilePath(
config.getOutputDirectory(), "summary", timestampString, sequentialId)
+ ".csv";
String extendedPath = ResultsWriter.getOutputFilePath(
config.getOutputDirectory(), "extended_summary", timestampString, sequentialId)
+ ".csv";

try (BufferedReader brSummary = new BufferedReader(new FileReader(summaryPath, UTF_8));
BufferedReader brExtended = new BufferedReader(new FileReader(extendedPath, UTF_8))) {
while (true) {
String lineExtended = brExtended.readLine();
// If the extended file has reached its end, then the non-extended file must have too
if (lineExtended == null) {
assertNull(brSummary.readLine(), "Extended file is missing a line");
return;
}

// If the extended file should be excluded, continue without moving the file pointer
// in the non-extended file. For now, there's only one type of row excluded, and they
// happen to all start with "Inactive Ballots by"
if (lineExtended.startsWith("Inactive Ballots by")) {
continue;
}

// This line should be equal in both files. Ensure the line exists and they're equal
// in both files.
String lineSummary = brSummary.readLine();
assertNotNull(lineSummary, "Summary file is missing a line");
if (!lineSummary.equals(lineExtended)) {
fail("Line differes in extended vs non-extended CSV: \n%s\n%s".formatted(
lineSummary, lineExtended));
}
}
} catch (FileNotFoundException exception) {
Logger.severe("File not found!\n%s", exception);
fail();
} catch (IOException exception) {
Logger.severe("Error reading file!\n%s", exception);
fail();
}
}

@BeforeAll
static void setup() {
Logger.setup();
Expand Down