Have dominion autoload use both names and codes (alt)

src/main/java/network/brightspots/rcv/BaseCvrReader.java

@@ -21,7 +21,10 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 import javafx.util.Pair;
+import network.brightspots.rcv.RawContestConfig.Candidate;
 import network.brightspots.rcv.RawContestConfig.CvrSource;
 
 abstract class BaseCvrReader {
@@ -63,19 +66,19 @@ public void runAdditionalValidations(List<CastVoteRecord> castVoteRecords)
   }
 
   // Some CVRs have a list of candidates in the file. Read that list and return it.
-  // This will be used in tandem with gatherUnknownCandidates, which only looks for candidates
+  // This will be used in tandem with gatherUnknownCandidateCounts, which only looks for candidates
   // that have at least one vote.
-  public List<String> readCandidateListFromCvr(List<CastVoteRecord> castVoteRecords)
+  public List<String> readCandidateListFromCvr()
       throws IOException {
     return new ArrayList<>();
   }
 
   // Gather candidate names from the CVR that are not in the config.
-  Map<String, Integer> gatherUnknownCandidates(
+  public Map<Candidate, Integer> gatherUnknownCandidateCounts(
       List<CastVoteRecord> castVoteRecords, boolean includeCandidatesWithZeroVotes) {
     // First pass: gather all unrecognized candidates and their counts
     // All CVR Readers have this implemented
-    Map<String, Integer> unrecognizedCandidateCounts = new HashMap<>();
+    Map<String, Integer> unrecognizedNameCounts = new HashMap<>();
     for (CastVoteRecord cvr : castVoteRecords) {
       for (Pair<Integer, CandidatesAtRanking> ranking : cvr.candidateRankings) {
         for (String candidateName : ranking.getValue()) {
@@ -85,7 +88,7 @@ Map<String, Integer> gatherUnknownCandidates(
             continue;
           }
 
-          unrecognizedCandidateCounts.merge(candidateName, 1, Integer::sum);
+          unrecognizedNameCounts.merge(candidateName, 1, Integer::sum);
         }
       }
     }
@@ -97,7 +100,7 @@ Map<String, Integer> gatherUnknownCandidates(
       // during auto-load candidates and just use readCandidateListFromCvr.
       List<String> allCandidates = new ArrayList<>();
       try {
-        allCandidates = readCandidateListFromCvr(castVoteRecords);
+        allCandidates = readCandidateListFromCvr();
       } catch (IOException e) {
         // If we can't read the candidate list, we can't check for unrecognized candidates.
         Logger.warning("IOException reading candidate list from CVR: %s", e.getMessage());
@@ -109,14 +112,22 @@ Map<String, Integer> gatherUnknownCandidates(
 
       // Combine the lists
       for (String candidateName : allCandidates) {
-        if (!unrecognizedCandidateCounts.containsKey(candidateName)
+        if (!unrecognizedNameCounts.containsKey(candidateName)
             && config.getNameForCandidate(candidateName) == null) {
-          unrecognizedCandidateCounts.put(candidateName, 0);
+          unrecognizedNameCounts.put(candidateName, 0);
         }
       }
     }
 
-    return unrecognizedCandidateCounts;
+    // Change the map to use Candidate objects instead of names
+    return unrecognizedNameCounts.entrySet().stream()
+          .collect(Collectors.toMap(entry -> new Candidate(entry.getKey()), Map.Entry::getValue));
+  }
+
+  Set<Candidate> gatherUnknownCandidates(List<CastVoteRecord> castVoteRecords)
+          throws CastVoteRecord.CvrParseException, IOException {
+    readCastVoteRecords(castVoteRecords);
+    return gatherUnknownCandidateCounts(castVoteRecords, true).keySet();
   }
 
   boolean usesLastAllowedRanking(List<Pair<Integer, String>> rankings, String contestId) {

src/main/java/network/brightspots/rcv/CsvCvrReader.java

@@ -48,7 +48,7 @@ public String readerName() {
   }
 
   @Override
-  public List<String> readCandidateListFromCvr(List<CastVoteRecord> castVoteRecords)
+  public List<String> readCandidateListFromCvr()
       throws IOException {
     try (FileInputStream inputStream = new FileInputStream(Path.of(cvrPath).toFile())) {
       return getCandidateNamesAndInitializeParser(getCsvParser(inputStream));

src/main/java/network/brightspots/rcv/DominionCvrReader.java

@@ -51,7 +51,7 @@ class DominionCvrReader extends BaseCvrReader {
   private Map<Integer, String> precinctPortions;
   // map of contest ID to Contest data
   private Map<String, Contest> contests;
-  private List<Candidate> candidates;
+  private Map<String, Candidate> candidateCodesToCandidates;
 
   DominionCvrReader(ContestConfig config, RawContestConfig.CvrSource source) {
     super(config, source);
@@ -100,9 +100,9 @@ private static Map<Integer, String> getPrecinctData(String precinctPath) {
     return precinctsById;
   }
 
-  // returns list of Candidate objects parsed from CandidateManifest.json
-  private static List<Candidate> getCandidates(String candidatePath) {
-    ArrayList<Candidate> candidates = new ArrayList<>();
+  // returns a map of Codes to Candidate objects parsed from CandidateManifest.json
+  private Map<String, Candidate> getCandidates(String candidatePath) {
+    Map<String, Candidate> candidateCodesToCandidates = new HashMap<>();
     try {
       HashMap json = JsonParser.readFromFile(candidatePath, HashMap.class);
       ArrayList candidateList = (ArrayList) json.get("List");
@@ -112,14 +112,17 @@ private static List<Candidate> getCandidates(String candidatePath) {
         Integer id = (Integer) candidateMap.get("Id");
         String code = id.toString();
         String contestId = candidateMap.get("ContestId").toString();
+        if (!source.getContestId().equals(contestId)) {
+          continue;
+        }
         Candidate newCandidate = new Candidate(name, code, contestId);
-        candidates.add(newCandidate);
+        candidateCodesToCandidates.put(code, newCandidate);
       }
     } catch (Exception exception) {
       Logger.severe("Error parsing candidate manifest:\n%s", exception);
-      candidates = null;
+      candidateCodesToCandidates = null;
     }
-    return candidates;
+    return candidateCodesToCandidates;
   }
 
   @Override
@@ -131,6 +134,16 @@ public String readerName() {
   // them to the input list
   @Override
   void readCastVoteRecords(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
+    loadManifests();
+
+    gatherCvrsForContest(castVoteRecords, source.getContestId());
+    if (castVoteRecords.isEmpty()) {
+      Logger.severe("No cast vote record data found!");
+      throw new CvrParseException();
+    }
+  }
+
+  private void loadManifests() throws CvrParseException {
     // read metadata files for precincts, precinct portions, contest, and candidates
 
     // Precinct data does not exist for earlier versions of Dominion (only precinct portion)
@@ -157,17 +170,11 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords) throws CvrParseEx
       throw new CvrParseException();
     }
     Path candidatePath = Paths.get(cvrPath, CANDIDATE_MANIFEST);
-    this.candidates = getCandidates(candidatePath.toString());
-    if (this.candidates == null) {
+    this.candidateCodesToCandidates = getCandidates(candidatePath.toString());
+    if (this.candidateCodesToCandidates == null) {
       Logger.severe("No candidate data found!");
       throw new CvrParseException();
     }
-    // parse the cvr file(s)
-    gatherCvrsForContest(castVoteRecords, source.getContestId());
-    if (castVoteRecords.isEmpty()) {
-      Logger.severe("No cast vote record data found!");
-      throw new CvrParseException();
-    }
   }
 
   @Override
@@ -188,22 +195,22 @@ private void validateNamesAreInContest(List<CastVoteRecord> castVoteRecords)
       throws CastVoteRecord.CvrParseException {
     // build a lookup map to optimize CVR parsing
     Map<String, Set<String>> contestIdToCandidateNames = new HashMap<>();
-    for (Candidate candidate : this.candidates) {
-      Set<String> candidates;
+    for (Candidate candidate : this.candidateCodesToCandidates.values()) {
+      Set<String> candidateNames;
       if (contestIdToCandidateNames.containsKey(candidate.getContestId())) {
-        candidates = contestIdToCandidateNames.get(candidate.getContestId());
+        candidateNames = contestIdToCandidateNames.get(candidate.getContestId());
       } else {
-        candidates = new HashSet<>();
+        candidateNames = new HashSet<>();
       }
-      candidates.add(config.getNameForCandidate(candidate.getCode()));
-      contestIdToCandidateNames.put(candidate.getContestId(), candidates);
+      candidateNames.add(config.getNameForCandidate(candidate.getCode()));
+      contestIdToCandidateNames.put(candidate.getContestId(), candidateNames);
     }
 
     // Check each candidate exists in the contest
     for (CastVoteRecord cvr : castVoteRecords) {
       String contestId = cvr.getContestId();
-      Set<String> candidates = contestIdToCandidateNames.get(contestId);
-      if (candidates == null) {
+      Set<String> candidateNames = contestIdToCandidateNames.get(contestId);
+      if (candidateNames == null) {
         Logger.severe("Contest ID '%s' had no candidates!", contestId);
         throw new CastVoteRecord.CvrParseException();
       }
@@ -216,7 +223,7 @@ private void validateNamesAreInContest(List<CastVoteRecord> castVoteRecords)
           if (candidateId.equals(Tabulator.UNDECLARED_WRITE_IN_OUTPUT_LABEL)) {
             continue;
           }
-          if (!candidates.contains(candidateName)) {
+          if (!candidateNames.contains(candidateName)) {
             Logger.severe(
                 "Candidate ID '%s' is not valid for contest '%s'!", candidateName, contestId);
             throw new CastVoteRecord.CvrParseException();
@@ -226,6 +233,28 @@ private void validateNamesAreInContest(List<CastVoteRecord> castVoteRecords)
     }
   }
 
+  // The Candidate Autoload looks only at the CVR file(s) and not the manifest files, so
+  // it doesn't know about the mapping between a code and the candidate's name. This function
+  // addresses that discrepancy, while also being much faster than actually reading each ballot.
+  @Override
+  public Set<RawContestConfig.Candidate> gatherUnknownCandidates(
+          List<CastVoteRecord> castVoteRecords) {
+    try {
+      loadManifests();
+    } catch (CvrParseException exception) {
+      Logger.severe("Error loading manifest data:\n%s", exception);
+      return new HashSet<>();
+    }
+
+    Set<String> knownNames = config.getCandidateNames();
+
+    // Return the candidate codes that are not in the knownNames set
+    return candidateCodesToCandidates.entrySet().stream()
+            .filter(entry -> !knownNames.contains(entry.getValue().name))
+            .map(entry -> new RawContestConfig.Candidate(entry.getValue().name, entry.getKey()))
+            .collect(Collectors.toSet());
+  }
+
   // parse the CVR file or files into a List of CastVoteRecords for tabulation
   private void gatherCvrsForContest(List<CastVoteRecord> castVoteRecords, String contestIdToLoad) {
     try {

src/main/java/network/brightspots/rcv/GuiConfigController.java

@@ -1720,16 +1720,15 @@ protected Void call() {
               }
               if (cvrsSpecified) {
                 // Gather unloaded names from each of the sources and place into the HashSet
-                Set<String> unloadedNames = new HashSet<>();
+                HashSet<Candidate> unloadedCandidates = new HashSet<>();
                 for (CvrSource source : sources) {
                   Provider provider = ContestConfig.getProvider(source);
                   try {
                     List<CastVoteRecord> castVoteRecords = new ArrayList<>();
                     BaseCvrReader reader = provider.constructReader(config, source);
-                    reader.readCastVoteRecords(castVoteRecords);
-                    Set<String> unknownCandidates = reader.gatherUnknownCandidates(
-                        castVoteRecords, true).keySet();
-                    unloadedNames.addAll(unknownCandidates);
+                    Set<Candidate> unknownCandidates =
+                            reader.gatherUnknownCandidates(castVoteRecords);
+                    unloadedCandidates.addAll(unknownCandidates);
                   } catch (ContestConfig.UnrecognizedProviderException e) {
                     Logger.severe(
                         "Unrecognized provider \"%s\" in source file \"%s\": %s",
@@ -1743,15 +1742,14 @@ protected Void call() {
 
                 // Validate each name and add to the table of candidates
                 int successCount = 0;
-                for (String name : unloadedNames) {
-                  Candidate candidate = new Candidate(name, null, false);
+                for (Candidate candidate : unloadedCandidates) {
                   Set<ValidationError> validationErrors =
                       ContestConfig.performBasicCandidateValidation(candidate);
                   if (validationErrors.isEmpty()) {
                     tableViewCandidates.getItems().add(candidate);
                     successCount++;
                   } else {
-                    Logger.severe("Failed to load candidate \"%s\"!", name);
+                    Logger.severe("Failed to load candidate \"%s\"!", candidate.getName());
                   }
                 }
 

src/main/java/network/brightspots/rcv/RawContestConfig.java

@@ -300,6 +300,14 @@ public static class Candidate {
 
     Candidate() {}
 
+    Candidate(String name) {
+      this(name, null, false);
+    }
+
+    Candidate(String name, String newlineSeparatedAliases) {
+      this(name, newlineSeparatedAliases, false);
+    }
+
     Candidate(String name, String newlineSeparatedAliases, boolean excluded) {
       this.name.setValue(name);
       this.excluded.setValue(excluded);

src/main/java/network/brightspots/rcv/TabulatorSession.java

@@ -39,6 +39,7 @@
 import network.brightspots.rcv.ContestConfig.UnrecognizedProviderException;
 import network.brightspots.rcv.FileUtils.UnableToCreateDirectoryException;
 import network.brightspots.rcv.OutputWriter.RoundSnapshotDataMissingException;
+import network.brightspots.rcv.RawContestConfig.Candidate;
 import network.brightspots.rcv.Tabulator.TabulationAbortedException;
 
 @SuppressWarnings("RedundantSuppression")
@@ -357,8 +358,8 @@ private LoadedCvrData parseCastVoteRecords(
                 source, reader, sourceIndex, startIndex, castVoteRecords.size() - 1));
 
         // Check for unrecognized candidates
-        Map<String, Integer> unrecognizedCandidateCounts =
-            reader.gatherUnknownCandidates(castVoteRecords, false);
+        Map<Candidate, Integer> unrecognizedCandidateCounts =
+            reader.gatherUnknownCandidateCounts(castVoteRecords, false);
 
         if (!unrecognizedCandidateCounts.isEmpty()) {
           throw new UnrecognizedCandidatesException(unrecognizedCandidateCounts);
@@ -440,9 +441,9 @@ private LoadedCvrData parseCastVoteRecords(
   static class UnrecognizedCandidatesException extends Exception {
 
     // count of how many times each unrecognized candidate was encountered during CVR parsing
-    final Map<String, Integer> candidateCounts;
+    final Map<Candidate, Integer> candidateCounts;
 
-    UnrecognizedCandidatesException(Map<String, Integer> candidateCounts) {
+    UnrecognizedCandidatesException(Map<Candidate, Integer> candidateCounts) {
       this.candidateCounts = candidateCounts;
     }
   }