From 6b1fffa135f5435bf951147c7d89c600b6bf5b2c Mon Sep 17 00:00:00 2001 From: Armin Samii Date: Mon, 27 Jan 2025 09:27:48 -0500 Subject: [PATCH] Have dominion autoload use both names and codes (alt) (#908) For Dominion, properly auto-load both names and Dominion GUIDs as codes --- .../brightspots/rcv/BaseCvrReader.java | 29 ++++--- .../network/brightspots/rcv/CsvCvrReader.java | 2 +- .../brightspots/rcv/DominionCvrReader.java | 77 +++++++++++++------ .../brightspots/rcv/GuiConfigController.java | 14 ++-- .../brightspots/rcv/RawContestConfig.java | 8 ++ .../brightspots/rcv/TabulatorSession.java | 9 ++- 6 files changed, 93 insertions(+), 46 deletions(-) diff --git a/src/main/java/network/brightspots/rcv/BaseCvrReader.java b/src/main/java/network/brightspots/rcv/BaseCvrReader.java index df7b30bc..57599138 100644 --- a/src/main/java/network/brightspots/rcv/BaseCvrReader.java +++ b/src/main/java/network/brightspots/rcv/BaseCvrReader.java @@ -21,7 +21,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import javafx.util.Pair; +import network.brightspots.rcv.RawContestConfig.Candidate; import network.brightspots.rcv.RawContestConfig.CvrSource; abstract class BaseCvrReader { @@ -63,19 +66,19 @@ public void runAdditionalValidations(List castVoteRecords) } // Some CVRs have a list of candidates in the file. Read that list and return it. - // This will be used in tandem with gatherUnknownCandidates, which only looks for candidates + // This will be used in tandem with gatherUnknownCandidateCounts, which only looks for candidates // that have at least one vote. - public List readCandidateListFromCvr(List castVoteRecords) + public List readCandidateListFromCvr() throws IOException { return new ArrayList<>(); } // Gather candidate names from the CVR that are not in the config. - Map gatherUnknownCandidates( + public Map gatherUnknownCandidateCounts( List castVoteRecords, boolean includeCandidatesWithZeroVotes) { // First pass: gather all unrecognized candidates and their counts // All CVR Readers have this implemented - Map unrecognizedCandidateCounts = new HashMap<>(); + Map unrecognizedNameCounts = new HashMap<>(); for (CastVoteRecord cvr : castVoteRecords) { for (Pair ranking : cvr.candidateRankings) { for (String candidateName : ranking.getValue()) { @@ -85,7 +88,7 @@ Map gatherUnknownCandidates( continue; } - unrecognizedCandidateCounts.merge(candidateName, 1, Integer::sum); + unrecognizedNameCounts.merge(candidateName, 1, Integer::sum); } } } @@ -97,7 +100,7 @@ Map gatherUnknownCandidates( // during auto-load candidates and just use readCandidateListFromCvr. List allCandidates = new ArrayList<>(); try { - allCandidates = readCandidateListFromCvr(castVoteRecords); + allCandidates = readCandidateListFromCvr(); } catch (IOException e) { // If we can't read the candidate list, we can't check for unrecognized candidates. Logger.warning("IOException reading candidate list from CVR: %s", e.getMessage()); @@ -109,14 +112,22 @@ Map gatherUnknownCandidates( // Combine the lists for (String candidateName : allCandidates) { - if (!unrecognizedCandidateCounts.containsKey(candidateName) + if (!unrecognizedNameCounts.containsKey(candidateName) && config.getNameForCandidate(candidateName) == null) { - unrecognizedCandidateCounts.put(candidateName, 0); + unrecognizedNameCounts.put(candidateName, 0); } } } - return unrecognizedCandidateCounts; + // Change the map to use Candidate objects instead of names + return unrecognizedNameCounts.entrySet().stream() + .collect(Collectors.toMap(entry -> new Candidate(entry.getKey()), Map.Entry::getValue)); + } + + Set gatherUnknownCandidates(List castVoteRecords) + throws CastVoteRecord.CvrParseException, IOException { + readCastVoteRecords(castVoteRecords); + return gatherUnknownCandidateCounts(castVoteRecords, true).keySet(); } boolean usesLastAllowedRanking(List> rankings, String contestId) { diff --git a/src/main/java/network/brightspots/rcv/CsvCvrReader.java b/src/main/java/network/brightspots/rcv/CsvCvrReader.java index b81100c1..8498a71b 100644 --- a/src/main/java/network/brightspots/rcv/CsvCvrReader.java +++ b/src/main/java/network/brightspots/rcv/CsvCvrReader.java @@ -48,7 +48,7 @@ public String readerName() { } @Override - public List readCandidateListFromCvr(List castVoteRecords) + public List readCandidateListFromCvr() throws IOException { try (FileInputStream inputStream = new FileInputStream(Path.of(cvrPath).toFile())) { return getCandidateNamesAndInitializeParser(getCsvParser(inputStream)); diff --git a/src/main/java/network/brightspots/rcv/DominionCvrReader.java b/src/main/java/network/brightspots/rcv/DominionCvrReader.java index bd4270c6..a8637c01 100644 --- a/src/main/java/network/brightspots/rcv/DominionCvrReader.java +++ b/src/main/java/network/brightspots/rcv/DominionCvrReader.java @@ -54,7 +54,7 @@ class DominionCvrReader extends BaseCvrReader { private Map precinctPortions; // map of contest ID to Contest data private Map contests; - private List candidates; + private Map candidateCodesToCandidates; DominionCvrReader(ContestConfig config, RawContestConfig.CvrSource source) { super(config, source); @@ -103,9 +103,9 @@ private static Map getPrecinctData(String precinctPath) { return precinctsById; } - // returns list of Candidate objects parsed from CandidateManifest.json - private static List getCandidates(String candidatePath) { - ArrayList candidates = new ArrayList<>(); + // returns a map of Codes to Candidate objects parsed from CandidateManifest.json + private Map getCandidates(String candidatePath) { + Map candidateCodesToCandidates = new HashMap<>(); try { HashMap json = JsonParser.readFromFile(candidatePath, HashMap.class); ArrayList candidateList = (ArrayList) json.get("List"); @@ -115,14 +115,17 @@ private static List getCandidates(String candidatePath) { Integer id = (Integer) candidateMap.get("Id"); String code = id.toString(); String contestId = candidateMap.get("ContestId").toString(); + if (!source.getContestId().equals(contestId)) { + continue; + } Candidate newCandidate = new Candidate(name, code, contestId); - candidates.add(newCandidate); + candidateCodesToCandidates.put(code, newCandidate); } } catch (Exception exception) { Logger.severe("Error parsing candidate manifest:\n%s", exception); - candidates = null; + candidateCodesToCandidates = null; } - return candidates; + return candidateCodesToCandidates; } @Override @@ -134,6 +137,16 @@ public String readerName() { // them to the input list @Override void readCastVoteRecords(List castVoteRecords) throws CvrParseException { + loadManifests(); + + gatherCvrsForContest(castVoteRecords, source.getContestId()); + if (castVoteRecords.isEmpty()) { + Logger.severe("No cast vote record data found!"); + throw new CvrParseException(); + } + } + + private void loadManifests() throws CvrParseException { // read metadata files for precincts, precinct portions, contest, and candidates // Precinct data does not exist for earlier versions of Dominion (only precinct portion) @@ -160,17 +173,11 @@ void readCastVoteRecords(List castVoteRecords) throws CvrParseEx throw new CvrParseException(); } Path candidatePath = Paths.get(cvrPath, CANDIDATE_MANIFEST); - this.candidates = getCandidates(candidatePath.toString()); - if (this.candidates == null) { + this.candidateCodesToCandidates = getCandidates(candidatePath.toString()); + if (this.candidateCodesToCandidates == null) { Logger.severe("No candidate data found!"); throw new CvrParseException(); } - // parse the cvr file(s) - gatherCvrsForContest(castVoteRecords, source.getContestId()); - if (castVoteRecords.isEmpty()) { - Logger.severe("No cast vote record data found!"); - throw new CvrParseException(); - } } @Override @@ -191,22 +198,22 @@ private void validateNamesAreInContest(List castVoteRecords) throws CastVoteRecord.CvrParseException { // build a lookup map to optimize CVR parsing Map> contestIdToCandidateNames = new HashMap<>(); - for (Candidate candidate : this.candidates) { - Set candidates; + for (Candidate candidate : this.candidateCodesToCandidates.values()) { + Set candidateNames; if (contestIdToCandidateNames.containsKey(candidate.getContestId())) { - candidates = contestIdToCandidateNames.get(candidate.getContestId()); + candidateNames = contestIdToCandidateNames.get(candidate.getContestId()); } else { - candidates = new HashSet<>(); + candidateNames = new HashSet<>(); } - candidates.add(config.getNameForCandidate(candidate.getCode())); - contestIdToCandidateNames.put(candidate.getContestId(), candidates); + candidateNames.add(config.getNameForCandidate(candidate.getCode())); + contestIdToCandidateNames.put(candidate.getContestId(), candidateNames); } // Check each candidate exists in the contest for (CastVoteRecord cvr : castVoteRecords) { String contestId = cvr.getContestId(); - Set candidates = contestIdToCandidateNames.get(contestId); - if (candidates == null) { + Set candidateNames = contestIdToCandidateNames.get(contestId); + if (candidateNames == null) { Logger.severe("Contest ID '%s' had no candidates!", contestId); throw new CastVoteRecord.CvrParseException(); } @@ -219,7 +226,7 @@ private void validateNamesAreInContest(List castVoteRecords) if (candidateId.equals(Tabulator.UNDECLARED_WRITE_IN_OUTPUT_LABEL)) { continue; } - if (!candidates.contains(candidateName)) { + if (!candidateNames.contains(candidateName)) { Logger.severe( "Candidate ID '%s' is not valid for contest '%s'!", candidateName, contestId); throw new CastVoteRecord.CvrParseException(); @@ -229,6 +236,28 @@ private void validateNamesAreInContest(List castVoteRecords) } } + // The Candidate Autoload looks only at the CVR file(s) and not the manifest files, so + // it doesn't know about the mapping between a code and the candidate's name. This function + // addresses that discrepancy, while also being much faster than actually reading each ballot. + @Override + public Set gatherUnknownCandidates( + List castVoteRecords) { + try { + loadManifests(); + } catch (CvrParseException exception) { + Logger.severe("Error loading manifest data:\n%s", exception); + return new HashSet<>(); + } + + Set knownNames = config.getCandidateNames(); + + // Return the candidate codes that are not in the knownNames set + return candidateCodesToCandidates.entrySet().stream() + .filter(entry -> !knownNames.contains(entry.getValue().name)) + .map(entry -> new RawContestConfig.Candidate(entry.getValue().name, entry.getKey())) + .collect(Collectors.toSet()); + } + // parse the CVR file or files into a List of CastVoteRecords for tabulation private void gatherCvrsForContest(List castVoteRecords, String contestIdToLoad) { try { diff --git a/src/main/java/network/brightspots/rcv/GuiConfigController.java b/src/main/java/network/brightspots/rcv/GuiConfigController.java index 4ccf4d1e..3ab19e37 100644 --- a/src/main/java/network/brightspots/rcv/GuiConfigController.java +++ b/src/main/java/network/brightspots/rcv/GuiConfigController.java @@ -1722,16 +1722,15 @@ protected Void call() { } if (cvrsSpecified) { // Gather unloaded names from each of the sources and place into the HashSet - Set unloadedNames = new HashSet<>(); + HashSet unloadedCandidates = new HashSet<>(); for (CvrSource source : sources) { Provider provider = ContestConfig.getProvider(source); try { List castVoteRecords = new ArrayList<>(); BaseCvrReader reader = provider.constructReader(config, source); - reader.readCastVoteRecords(castVoteRecords); - Set unknownCandidates = reader.gatherUnknownCandidates( - castVoteRecords, true).keySet(); - unloadedNames.addAll(unknownCandidates); + Set unknownCandidates = + reader.gatherUnknownCandidates(castVoteRecords); + unloadedCandidates.addAll(unknownCandidates); } catch (ContestConfig.UnrecognizedProviderException e) { Logger.severe( "Unrecognized provider \"%s\" in source file \"%s\": %s", @@ -1745,15 +1744,14 @@ protected Void call() { // Validate each name and add to the table of candidates int successCount = 0; - for (String name : unloadedNames) { - Candidate candidate = new Candidate(name, null, false); + for (Candidate candidate : unloadedCandidates) { Set validationErrors = ContestConfig.performBasicCandidateValidation(candidate); if (validationErrors.isEmpty()) { tableViewCandidates.getItems().add(candidate); successCount++; } else { - Logger.severe("Failed to load candidate \"%s\"!", name); + Logger.severe("Failed to load candidate \"%s\"!", candidate.getName()); } } diff --git a/src/main/java/network/brightspots/rcv/RawContestConfig.java b/src/main/java/network/brightspots/rcv/RawContestConfig.java index d297e6ef..2de85f21 100644 --- a/src/main/java/network/brightspots/rcv/RawContestConfig.java +++ b/src/main/java/network/brightspots/rcv/RawContestConfig.java @@ -300,6 +300,14 @@ public static class Candidate { Candidate() {} + Candidate(String name) { + this(name, null, false); + } + + Candidate(String name, String newlineSeparatedAliases) { + this(name, newlineSeparatedAliases, false); + } + Candidate(String name, String newlineSeparatedAliases, boolean excluded) { this.name.setValue(name); this.excluded.setValue(excluded); diff --git a/src/main/java/network/brightspots/rcv/TabulatorSession.java b/src/main/java/network/brightspots/rcv/TabulatorSession.java index 86bd362f..ce81d956 100644 --- a/src/main/java/network/brightspots/rcv/TabulatorSession.java +++ b/src/main/java/network/brightspots/rcv/TabulatorSession.java @@ -39,6 +39,7 @@ import network.brightspots.rcv.ContestConfig.UnrecognizedProviderException; import network.brightspots.rcv.FileUtils.UnableToCreateDirectoryException; import network.brightspots.rcv.OutputWriter.RoundSnapshotDataMissingException; +import network.brightspots.rcv.RawContestConfig.Candidate; import network.brightspots.rcv.Tabulator.TabulationAbortedException; @SuppressWarnings("RedundantSuppression") @@ -357,8 +358,8 @@ private LoadedCvrData parseCastVoteRecords( source, reader, sourceIndex, startIndex, castVoteRecords.size() - 1)); // Check for unrecognized candidates - Map unrecognizedCandidateCounts = - reader.gatherUnknownCandidates(castVoteRecords, false); + Map unrecognizedCandidateCounts = + reader.gatherUnknownCandidateCounts(castVoteRecords, false); if (!unrecognizedCandidateCounts.isEmpty()) { throw new UnrecognizedCandidatesException(unrecognizedCandidateCounts); @@ -440,9 +441,9 @@ private LoadedCvrData parseCastVoteRecords( static class UnrecognizedCandidatesException extends Exception { // count of how many times each unrecognized candidate was encountered during CVR parsing - final Map candidateCounts; + final Map candidateCounts; - UnrecognizedCandidatesException(Map candidateCounts) { + UnrecognizedCandidatesException(Map candidateCounts) { this.candidateCounts = candidateCounts; } }