Skip to content

Commit

Permalink
Have dominion autoload use both names and codes (alt) (#908)
Browse files Browse the repository at this point in the history 
For Dominion, properly auto-load both names and Dominion GUIDs as codes
  • Loading branch information
@artoonie
artoonie authored Jan 27, 2025
1 parent eaaa4b6 commit 6b1fffa
Show file tree
Hide file tree
Showing 6 changed files with 93 additions and 46 deletions.
29 changes: 20 additions & 9 deletions src/main/java/network/brightspots/rcv/BaseCvrReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import javafx.util.Pair;
import network.brightspots.rcv.RawContestConfig.Candidate;
import network.brightspots.rcv.RawContestConfig.CvrSource;

abstract class BaseCvrReader {
Expand Down Expand Up @@ -63,19 +66,19 @@ public void runAdditionalValidations(List<CastVoteRecord> castVoteRecords)
}

// Some CVRs have a list of candidates in the file. Read that list and return it.
// This will be used in tandem with gatherUnknownCandidates, which only looks for candidates
// This will be used in tandem with gatherUnknownCandidateCounts, which only looks for candidates
// that have at least one vote.
public List<String> readCandidateListFromCvr(List<CastVoteRecord> castVoteRecords)
public List<String> readCandidateListFromCvr()
throws IOException {
return new ArrayList<>();
}

// Gather candidate names from the CVR that are not in the config.
Map<String, Integer> gatherUnknownCandidates(
public Map<Candidate, Integer> gatherUnknownCandidateCounts(
List<CastVoteRecord> castVoteRecords, boolean includeCandidatesWithZeroVotes) {
// First pass: gather all unrecognized candidates and their counts
// All CVR Readers have this implemented
Map<String, Integer> unrecognizedCandidateCounts = new HashMap<>();
Map<String, Integer> unrecognizedNameCounts = new HashMap<>();
for (CastVoteRecord cvr : castVoteRecords) {
for (Pair<Integer, CandidatesAtRanking> ranking : cvr.candidateRankings) {
for (String candidateName : ranking.getValue()) {
Expand All @@ -85,7 +88,7 @@ Map<String, Integer> gatherUnknownCandidates(
continue;
}

unrecognizedCandidateCounts.merge(candidateName, 1, Integer::sum);
unrecognizedNameCounts.merge(candidateName, 1, Integer::sum);
}
}
}
Expand All @@ -97,7 +100,7 @@ Map<String, Integer> gatherUnknownCandidates(
// during auto-load candidates and just use readCandidateListFromCvr.
List<String> allCandidates = new ArrayList<>();
try {
allCandidates = readCandidateListFromCvr(castVoteRecords);
allCandidates = readCandidateListFromCvr();
} catch (IOException e) {
// If we can't read the candidate list, we can't check for unrecognized candidates.
Logger.warning("IOException reading candidate list from CVR: %s", e.getMessage());
Expand All @@ -109,14 +112,22 @@ Map<String, Integer> gatherUnknownCandidates(

// Combine the lists
for (String candidateName : allCandidates) {
if (!unrecognizedCandidateCounts.containsKey(candidateName)
if (!unrecognizedNameCounts.containsKey(candidateName)
&& config.getNameForCandidate(candidateName) == null) {
unrecognizedCandidateCounts.put(candidateName, 0);
unrecognizedNameCounts.put(candidateName, 0);
}
}
}

return unrecognizedCandidateCounts;
// Change the map to use Candidate objects instead of names
return unrecognizedNameCounts.entrySet().stream()
.collect(Collectors.toMap(entry -> new Candidate(entry.getKey()), Map.Entry::getValue));
}

Set<Candidate> gatherUnknownCandidates(List<CastVoteRecord> castVoteRecords)
throws CastVoteRecord.CvrParseException, IOException {
readCastVoteRecords(castVoteRecords);
return gatherUnknownCandidateCounts(castVoteRecords, true).keySet();
}

boolean usesLastAllowedRanking(List<Pair<Integer, String>> rankings, String contestId) {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/network/brightspots/rcv/CsvCvrReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public String readerName() {
}

@Override
public List<String> readCandidateListFromCvr(List<CastVoteRecord> castVoteRecords)
public List<String> readCandidateListFromCvr()
throws IOException {
try (FileInputStream inputStream = new FileInputStream(Path.of(cvrPath).toFile())) {
return getCandidateNamesAndInitializeParser(getCsvParser(inputStream));
Expand Down
77 changes: 53 additions & 24 deletions src/main/java/network/brightspots/rcv/DominionCvrReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class DominionCvrReader extends BaseCvrReader {
private Map<Integer, String> precinctPortions;
// map of contest ID to Contest data
private Map<String, Contest> contests;
private List<Candidate> candidates;
private Map<String, Candidate> candidateCodesToCandidates;

DominionCvrReader(ContestConfig config, RawContestConfig.CvrSource source) {
super(config, source);
Expand Down Expand Up @@ -103,9 +103,9 @@ private static Map<Integer, String> getPrecinctData(String precinctPath) {
return precinctsById;
}

// returns list of Candidate objects parsed from CandidateManifest.json
private static List<Candidate> getCandidates(String candidatePath) {
ArrayList<Candidate> candidates = new ArrayList<>();
// returns a map of Codes to Candidate objects parsed from CandidateManifest.json
private Map<String, Candidate> getCandidates(String candidatePath) {
Map<String, Candidate> candidateCodesToCandidates = new HashMap<>();
try {
HashMap json = JsonParser.readFromFile(candidatePath, HashMap.class);
ArrayList candidateList = (ArrayList) json.get("List");
Expand All @@ -115,14 +115,17 @@ private static List<Candidate> getCandidates(String candidatePath) {
Integer id = (Integer) candidateMap.get("Id");
String code = id.toString();
String contestId = candidateMap.get("ContestId").toString();
if (!source.getContestId().equals(contestId)) {
continue;
}
Candidate newCandidate = new Candidate(name, code, contestId);
candidates.add(newCandidate);
candidateCodesToCandidates.put(code, newCandidate);
}
} catch (Exception exception) {
Logger.severe("Error parsing candidate manifest:\n%s", exception);
candidates = null;
candidateCodesToCandidates = null;
}
return candidates;
return candidateCodesToCandidates;
}

@Override
Expand All @@ -134,6 +137,16 @@ public String readerName() {
// them to the input list
@Override
void readCastVoteRecords(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
loadManifests();

gatherCvrsForContest(castVoteRecords, source.getContestId());
if (castVoteRecords.isEmpty()) {
Logger.severe("No cast vote record data found!");
throw new CvrParseException();
}
}

private void loadManifests() throws CvrParseException {
// read metadata files for precincts, precinct portions, contest, and candidates

// Precinct data does not exist for earlier versions of Dominion (only precinct portion)
Expand All @@ -160,17 +173,11 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords) throws CvrParseEx
throw new CvrParseException();
}
Path candidatePath = Paths.get(cvrPath, CANDIDATE_MANIFEST);
this.candidates = getCandidates(candidatePath.toString());
if (this.candidates == null) {
this.candidateCodesToCandidates = getCandidates(candidatePath.toString());
if (this.candidateCodesToCandidates == null) {
Logger.severe("No candidate data found!");
throw new CvrParseException();
}
// parse the cvr file(s)
gatherCvrsForContest(castVoteRecords, source.getContestId());
if (castVoteRecords.isEmpty()) {
Logger.severe("No cast vote record data found!");
throw new CvrParseException();
}
}

@Override
Expand All @@ -191,22 +198,22 @@ private void validateNamesAreInContest(List<CastVoteRecord> castVoteRecords)
throws CastVoteRecord.CvrParseException {
// build a lookup map to optimize CVR parsing
Map<String, Set<String>> contestIdToCandidateNames = new HashMap<>();
for (Candidate candidate : this.candidates) {
Set<String> candidates;
for (Candidate candidate : this.candidateCodesToCandidates.values()) {
Set<String> candidateNames;
if (contestIdToCandidateNames.containsKey(candidate.getContestId())) {
candidates = contestIdToCandidateNames.get(candidate.getContestId());
candidateNames = contestIdToCandidateNames.get(candidate.getContestId());
} else {
candidates = new HashSet<>();
candidateNames = new HashSet<>();
}
candidates.add(config.getNameForCandidate(candidate.getCode()));
contestIdToCandidateNames.put(candidate.getContestId(), candidates);
candidateNames.add(config.getNameForCandidate(candidate.getCode()));
contestIdToCandidateNames.put(candidate.getContestId(), candidateNames);
}

// Check each candidate exists in the contest
for (CastVoteRecord cvr : castVoteRecords) {
String contestId = cvr.getContestId();
Set<String> candidates = contestIdToCandidateNames.get(contestId);
if (candidates == null) {
Set<String> candidateNames = contestIdToCandidateNames.get(contestId);
if (candidateNames == null) {
Logger.severe("Contest ID '%s' had no candidates!", contestId);
throw new CastVoteRecord.CvrParseException();
}
Expand All @@ -219,7 +226,7 @@ private void validateNamesAreInContest(List<CastVoteRecord> castVoteRecords)
if (candidateId.equals(Tabulator.UNDECLARED_WRITE_IN_OUTPUT_LABEL)) {
continue;
}
if (!candidates.contains(candidateName)) {
if (!candidateNames.contains(candidateName)) {
Logger.severe(
"Candidate ID '%s' is not valid for contest '%s'!", candidateName, contestId);
throw new CastVoteRecord.CvrParseException();
Expand All @@ -229,6 +236,28 @@ private void validateNamesAreInContest(List<CastVoteRecord> castVoteRecords)
}
}

// The Candidate Autoload looks only at the CVR file(s) and not the manifest files, so
// it doesn't know about the mapping between a code and the candidate's name. This function
// addresses that discrepancy, while also being much faster than actually reading each ballot.
@Override
public Set<RawContestConfig.Candidate> gatherUnknownCandidates(
List<CastVoteRecord> castVoteRecords) {
try {
loadManifests();
} catch (CvrParseException exception) {
Logger.severe("Error loading manifest data:\n%s", exception);
return new HashSet<>();
}

Set<String> knownNames = config.getCandidateNames();

// Return the candidate codes that are not in the knownNames set
return candidateCodesToCandidates.entrySet().stream()
.filter(entry -> !knownNames.contains(entry.getValue().name))
.map(entry -> new RawContestConfig.Candidate(entry.getValue().name, entry.getKey()))
.collect(Collectors.toSet());
}

// parse the CVR file or files into a List of CastVoteRecords for tabulation
private void gatherCvrsForContest(List<CastVoteRecord> castVoteRecords, String contestIdToLoad) {
try {
Expand Down
14 changes: 6 additions & 8 deletions src/main/java/network/brightspots/rcv/GuiConfigController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1722,16 +1722,15 @@ protected Void call() {
}
if (cvrsSpecified) {
// Gather unloaded names from each of the sources and place into the HashSet
Set<String> unloadedNames = new HashSet<>();
HashSet<Candidate> unloadedCandidates = new HashSet<>();
for (CvrSource source : sources) {
Provider provider = ContestConfig.getProvider(source);
try {
List<CastVoteRecord> castVoteRecords = new ArrayList<>();
BaseCvrReader reader = provider.constructReader(config, source);
reader.readCastVoteRecords(castVoteRecords);
Set<String> unknownCandidates = reader.gatherUnknownCandidates(
castVoteRecords, true).keySet();
unloadedNames.addAll(unknownCandidates);
Set<Candidate> unknownCandidates =
reader.gatherUnknownCandidates(castVoteRecords);
unloadedCandidates.addAll(unknownCandidates);
} catch (ContestConfig.UnrecognizedProviderException e) {
Logger.severe(
"Unrecognized provider \"%s\" in source file \"%s\": %s",
Expand All @@ -1745,15 +1744,14 @@ protected Void call() {

// Validate each name and add to the table of candidates
int successCount = 0;
for (String name : unloadedNames) {
Candidate candidate = new Candidate(name, null, false);
for (Candidate candidate : unloadedCandidates) {
Set<ValidationError> validationErrors =
ContestConfig.performBasicCandidateValidation(candidate);
if (validationErrors.isEmpty()) {
tableViewCandidates.getItems().add(candidate);
successCount++;
} else {
Logger.severe("Failed to load candidate \"%s\"!", name);
Logger.severe("Failed to load candidate \"%s\"!", candidate.getName());
}
}

Expand Down
8 changes: 8 additions & 0 deletions src/main/java/network/brightspots/rcv/RawContestConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,14 @@ public static class Candidate {

Candidate() {}

Candidate(String name) {
this(name, null, false);
}

Candidate(String name, String newlineSeparatedAliases) {
this(name, newlineSeparatedAliases, false);
}

Candidate(String name, String newlineSeparatedAliases, boolean excluded) {
this.name.setValue(name);
this.excluded.setValue(excluded);
Expand Down
9 changes: 5 additions & 4 deletions src/main/java/network/brightspots/rcv/TabulatorSession.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import network.brightspots.rcv.ContestConfig.UnrecognizedProviderException;
import network.brightspots.rcv.FileUtils.UnableToCreateDirectoryException;
import network.brightspots.rcv.OutputWriter.RoundSnapshotDataMissingException;
import network.brightspots.rcv.RawContestConfig.Candidate;
import network.brightspots.rcv.Tabulator.TabulationAbortedException;

@SuppressWarnings("RedundantSuppression")
Expand Down Expand Up @@ -357,8 +358,8 @@ private LoadedCvrData parseCastVoteRecords(
source, reader, sourceIndex, startIndex, castVoteRecords.size() - 1));

// Check for unrecognized candidates
Map<String, Integer> unrecognizedCandidateCounts =
reader.gatherUnknownCandidates(castVoteRecords, false);
Map<Candidate, Integer> unrecognizedCandidateCounts =
reader.gatherUnknownCandidateCounts(castVoteRecords, false);

if (!unrecognizedCandidateCounts.isEmpty()) {
throw new UnrecognizedCandidatesException(unrecognizedCandidateCounts);
Expand Down Expand Up @@ -440,9 +441,9 @@ private LoadedCvrData parseCastVoteRecords(
static class UnrecognizedCandidatesException extends Exception {

// count of how many times each unrecognized candidate was encountered during CVR parsing
final Map<String, Integer> candidateCounts;
final Map<Candidate, Integer> candidateCounts;

UnrecognizedCandidatesException(Map<String, Integer> candidateCounts) {
UnrecognizedCandidatesException(Map<Candidate, Integer> candidateCounts) {
this.candidateCounts = candidateCounts;
}
}
Expand Down

0 comments on commit 6b1fffa

Please sign in to comment.