Improvements to Inactive Ballot tracking, and corrected reporting of undervotes

src/main/java/network/brightspots/rcv/CastVoteRecord.java

@@ -36,6 +36,8 @@ class CastVoteRecord {
   private final String precinct;
   // which precinct portion this ballot came from
   private final String precinctPortion;
+  // is the last-used ranking the last-allowed ranking in the CVR?
+  private final boolean usesLastAllowedRanking;
   // records winners to whom some fraction of this vote has been allocated
   private final Map<String, BigDecimal> winnerToFractionalValue = new HashMap<>();
   // If CVR CDF output is enabled, we store the necessary info here: for each round, the list of
@@ -65,8 +67,10 @@ class CastVoteRecord {
       String suppliedId,
       String precinct,
       String precinctPortion,
+      boolean usesLastAllowedRanking,
       List<Pair<Integer, String>> rankings) {
-    this(contestId, tabulatorId, batchId, suppliedId, null, precinct, precinctPortion, rankings);
+    this(contestId, tabulatorId, batchId, suppliedId, null, precinct, precinctPortion,
+        usesLastAllowedRanking, rankings);
   }
 
   CastVoteRecord(
@@ -77,6 +81,7 @@ class CastVoteRecord {
           String computedId,
           String precinct,
           String precinctPortion,
+          boolean usesLastAllowedRanking,
           List<Pair<Integer, String>> rankings) {
     this.contestId = contestId;
     this.tabulatorId = tabulatorId;
@@ -85,6 +90,7 @@ class CastVoteRecord {
     this.computedId = computedId;
     this.precinct = precinct;
     this.precinctPortion = precinctPortion;
+    this.usesLastAllowedRanking = usesLastAllowedRanking;
     this.candidateRankings = new CandidateRankingsList(rankings);
   }
 
@@ -93,8 +99,10 @@ class CastVoteRecord {
       String suppliedId,
       String precinct,
       String batchId,
+      boolean usesLastAllowedRanking,
       List<Pair<Integer, String>> rankings) {
-    this(null, null, batchId, suppliedId, computedId, precinct, null, rankings);
+    this(null, null, batchId, suppliedId, computedId, precinct, null,
+        usesLastAllowedRanking, rankings);
   }
 
   String getContestId() {
@@ -116,6 +124,10 @@ String getPrecinctPortion() {
     return precinctPortion;
   }
 
+  boolean doesUseLastAllowedRanking() {
+    return usesLastAllowedRanking;
+  }
+
   String getId() {
     return suppliedId != null ? suppliedId : computedId;
   }
@@ -224,11 +236,11 @@ Map<String, BigDecimal> getWinnerToFractionalValue() {
   // it is active or not.
   enum StatusForRound {
     ACTIVE,
-    INACTIVE_BY_UNDERVOTE,
-    INACTIVE_BY_OVERVOTE,
-    INACTIVE_BY_SKIPPED_RANKING,
-    INACTIVE_BY_REPEATED_RANKING,
-    INACTIVE_BY_EXHAUSTED_CHOICES
+    DID_NOT_RANK_ANY_CANDIDATES,
+    EXHAUSTED_CHOICE,
+    INVALIDATED_BY_OVERVOTE,
+    INVALIDATED_BY_SKIPPED_RANKING,
+    INVALIDATED_BY_REPEATED_RANKING,
   }
 
   enum VoteOutcomeType {

src/main/java/network/brightspots/rcv/ClearBallotCvrReader.java

@@ -102,6 +102,7 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
           }
         }
         // create the cast vote record
+        boolean usesLastAllowedRanking = !isRankingAllowed(rankings.size() + 1, null);
         CastVoteRecord castVoteRecord =
             new CastVoteRecord(
                 source.getContestId(),
@@ -110,6 +111,7 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
                 cvrData[CvrColumnField.BallotID.ordinal()],
                 cvrData[CvrColumnField.PrecinctID.ordinal()],
                 null,
+                usesLastAllowedRanking,
                 rankings);
 
         castVoteRecords.add(castVoteRecord);

src/main/java/network/brightspots/rcv/CommonDataFormatReader.java

@@ -315,8 +315,14 @@ void parseXml(List<CastVoteRecord> castVoteRecords) throws CvrParseException, IO
 
         String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
         // create the new CastVoteRecord
+        boolean usesLastAllowedRanking = isRankingAllowed(rankings.size() + 1, null);
         CastVoteRecord newRecord = new CastVoteRecord(
-            computedCastVoteRecordId, cvr.UniqueId, precinctId, cvr.BatchSequenceId, rankings);
+            computedCastVoteRecordId,
+            cvr.UniqueId,
+            precinctId,
+            cvr.BatchSequenceId,
+            usesLastAllowedRanking,
+            rankings);
         castVoteRecords.add(newRecord);
 
         // provide some user feedback on the CVR count
@@ -527,8 +533,14 @@ void parseJson(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
       String batchId = (String) cvr.get("BatchSequenceId");
       String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
       // create the new CastVoteRecord
-      CastVoteRecord newRecord =
-          new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, batchId, rankings);
+      boolean usesLastAllowedRanking = !isRankingAllowed(rankings.size() + 1, null);
+      CastVoteRecord newRecord = new CastVoteRecord(
+          computedCastVoteRecordId,
+          ballotId,
+          precinctId,
+          batchId,
+          usesLastAllowedRanking,
+          rankings);
       castVoteRecords.add(newRecord);
       // provide some user feedback on the CVR count
       if (castVoteRecords.size() % 50000 == 0) {

src/main/java/network/brightspots/rcv/CsvCvrReader.java

@@ -99,9 +99,14 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
         }
 
         // create the new CastVoteRecord
-        CastVoteRecord newCvr =
-            new CastVoteRecord(
-                Integer.toString(index), "no supplied ID", "no precinct", "no batch ID", rankings);
+        boolean usesLastAllowedRanking = !isRankingAllowed(rankings.size() + 1, null);
+        CastVoteRecord newCvr = new CastVoteRecord(
+            Integer.toString(index),
+            "no supplied ID",
+            "no precinct",
+            "no batch ID",
+            usesLastAllowedRanking,
+            rankings);
         castVoteRecords.add(newCvr);
       }
     } catch (IOException exception) {

src/main/java/network/brightspots/rcv/DominionCvrReader.java

@@ -357,9 +357,10 @@ private int parseCvrFile(
             rankings.add(ranking);
           }
           // create the new cvr
+          boolean usesLastAllowedRanking = !isRankingAllowed(rankings.size() + 1, contestId);
           CastVoteRecord newCvr =
               new CastVoteRecord(contestId, tabulatorId, batchId, suppliedId,
-                      computedId, precinct, precinctPortion, rankings);
+                      computedId, precinct, precinctPortion, usesLastAllowedRanking, rankings);
           castVoteRecords.add(newCvr);
         }
       }

src/main/java/network/brightspots/rcv/HartCvrReader.java

@@ -127,6 +127,7 @@ private void readCastVoteRecord(List<CastVoteRecord> castVoteRecords, Path path)
           }
         }
 
+        boolean usesLastAllowedRanking = !isRankingAllowed(rankings.size() + 1, null);
         CastVoteRecord cvr =
             new CastVoteRecord(
                 contest.Id,
@@ -135,6 +136,7 @@ private void readCastVoteRecord(List<CastVoteRecord> castVoteRecords, Path path)
                 xmlCvr.CvrGuid,
                 xmlCvr.PrecinctSplit.Name,
                 xmlCvr.PrecinctSplit.Id,
+                usesLastAllowedRanking,
                 rankings);
         castVoteRecords.add(cvr);
 

src/main/java/network/brightspots/rcv/ResultsWriter.java

@@ -374,18 +374,21 @@ private void generateSummarySpreadsheet(
     }
     csvPrinter.println();
 
-    Pair<String, StatusForRound>[] statusesToPrint =
-        new Pair[] {
-          new Pair<>("Overvotes", StatusForRound.INACTIVE_BY_OVERVOTE),
-          new Pair<>("Skipped Rankings", StatusForRound.INACTIVE_BY_SKIPPED_RANKING),
-          new Pair<>("Exhausted Choices", StatusForRound.INACTIVE_BY_EXHAUSTED_CHOICES),
-          new Pair<>("Repeated Rankings", StatusForRound.INACTIVE_BY_REPEATED_RANKING)
-        };
+    List<Pair<String, StatusForRound>> statusesToPrint = new ArrayList<>();
+    statusesToPrint.add(new Pair<>("Overvotes",
+            StatusForRound.INVALIDATED_BY_OVERVOTE));
+    statusesToPrint.add(new Pair<>("Skipped Rankings",
+            StatusForRound.INVALIDATED_BY_SKIPPED_RANKING));
+    statusesToPrint.add(new Pair<>("Exhausted Choices",
+            StatusForRound.EXHAUSTED_CHOICE));
+    statusesToPrint.add(new Pair<>("Repeated Rankings",
+            StatusForRound.INVALIDATED_BY_REPEATED_RANKING));
 
     for (Pair<String, StatusForRound> statusToPrint : statusesToPrint) {
       csvPrinter.print("Inactive Ballots by " + statusToPrint.getKey());
+
+      StatusForRound status = statusToPrint.getValue();
       for (int round = 1; round <= numRounds; round++) {
-        StatusForRound status = statusToPrint.getValue();
         BigDecimal thisRoundInactive = roundTallies.get(round).getBallotStatusTally(status);
         csvPrinter.print(thisRoundInactive);
 
@@ -405,14 +408,11 @@ private void generateSummarySpreadsheet(
     }
 
     csvPrinter.print("Inactive Ballots Total");
-    // Undervotes should not be included in the Inactive Ballots count, even though we treat them
-    // as such internally. Subtract undervotes (which are static throughout a contest) from the
-    // inactive ballot totals.
-    BigDecimal numUndervotes =
-        roundTallies.get(1).getBallotStatusTally(StatusForRound.INACTIVE_BY_UNDERVOTE);
+    BigDecimal numNoRankings =
+        roundTallies.get(1).getBallotStatusTally(StatusForRound.DID_NOT_RANK_ANY_CANDIDATES);
     for (int round = 1; round <= numRounds; round++) {
       BigDecimal thisRoundInactive = roundTallies.get(round).inactiveBallotSum();
-      csvPrinter.print(thisRoundInactive.subtract(numUndervotes));
+      csvPrinter.print(thisRoundInactive.subtract(numNoRankings));
 
       // Don't display percentage of inactive ballots
       csvPrinter.print("");
@@ -469,15 +469,15 @@ private void addActionRows(CSVPrinter csvPrinter) throws IOException {
 
   private void addContestSummaryRows(CSVPrinter csvPrinter, RoundTally round1Tally)
       throws IOException {
-    BigDecimal numUndervotes =
-        round1Tally.getBallotStatusTally(StatusForRound.INACTIVE_BY_UNDERVOTE);
+    BigDecimal numNoRankings =
+        round1Tally.getBallotStatusTally(StatusForRound.DID_NOT_RANK_ANY_CANDIDATES);
     BigDecimal totalNumberBallots =
         round1Tally.activeBallotSum().add(round1Tally.inactiveBallotSum());
     csvPrinter.printRecord("Contest Summary");
     csvPrinter.printRecord("Number to be Elected", config.getNumberOfWinners());
     csvPrinter.printRecord("Number of Candidates", config.getNumCandidates());
     csvPrinter.printRecord("Total Number of Ballots", totalNumberBallots);
-    csvPrinter.printRecord("Number of Undervotes", numUndervotes);
+    csvPrinter.printRecord("Number of Undervotes (No Rankings)", numNoRankings);
     csvPrinter.println();
   }
 
@@ -939,8 +939,8 @@ private void generateSummaryJson(
       configData.put(slice.toLowerString(), sliceId);
     }
 
-    BigDecimal firstRoundUndervotes =
-        roundTallies.get(1).getBallotStatusTally(StatusForRound.INACTIVE_BY_UNDERVOTE);
+    BigDecimal numNoRankings =
+        roundTallies.get(1).getBallotStatusTally(StatusForRound.DID_NOT_RANK_ANY_CANDIDATES);
     BigDecimal totalNumberBallots =
         roundTallies.get(1).activeBallotSum().add(roundTallies.get(1).inactiveBallotSum());
     BigDecimal lastRoundThreshold = roundTallies.get(numRounds).getWinningThreshold();
@@ -950,7 +950,7 @@ private void generateSummaryJson(
     summaryData.put("numWinners", config.getNumberOfWinners());
     summaryData.put("numCandidates", config.getCandidateNames().size());
     summaryData.put("totalNumBallots", totalNumberBallots);
-    summaryData.put("undervotes", firstRoundUndervotes.toBigInteger());
+    summaryData.put("undervotes", numNoRankings.toBigInteger());
 
     ArrayList<Object> results = new ArrayList<>();
     for (int round = 1; round <= numRounds; round++) {
@@ -992,10 +992,14 @@ private Map<String, BigDecimal> getInactiveJsonMap(RoundTally roundTally) {
     Map<String, BigDecimal> inactiveMap = new HashMap<>();
     Pair<String, StatusForRound>[] statusesToPrint =
         new Pair[] {
-          new Pair<>("overvotes", StatusForRound.INACTIVE_BY_OVERVOTE),
-          new Pair<>("skippedRankings", StatusForRound.INACTIVE_BY_SKIPPED_RANKING),
-          new Pair<>("exhaustedChoices", StatusForRound.INACTIVE_BY_EXHAUSTED_CHOICES),
-          new Pair<>("repeatedRankings", StatusForRound.INACTIVE_BY_REPEATED_RANKING)
+          new Pair<>("overvotes",
+                     StatusForRound.INVALIDATED_BY_OVERVOTE),
+          new Pair<>("skippedRankings",
+                     StatusForRound.INVALIDATED_BY_SKIPPED_RANKING),
+          new Pair<>("repeatedRankings",
+                     StatusForRound.INVALIDATED_BY_REPEATED_RANKING),
+          new Pair<>("exhaustedChoices",
+                     StatusForRound.EXHAUSTED_CHOICE),
         };
     for (Pair<String, StatusForRound> statusToPrint : statusesToPrint) {
       inactiveMap.put(

src/main/java/network/brightspots/rcv/StreamingCvrReader.java

@@ -216,11 +216,13 @@ private void endCvr() {
     Logger.auditable("[Raw Data]: " + currentCvrData.toString());
 
     // create new cast vote record
+    boolean usesLastAllowedRanking = !isRankingAllowed(currentRankings.size() + 1, null);
     CastVoteRecord newRecord = new CastVoteRecord(
         computedCastVoteRecordId,
         currentSuppliedCvrId,
         currentPrecinct,
         currentBatch,
+        usesLastAllowedRanking,
         currentRankings);
     cvrList.add(newRecord);
 

src/main/java/network/brightspots/rcv/Tabulator.java

@@ -1002,14 +1002,16 @@ private void recordSelectionForCastVoteRecord(
     String outcomeDescription;
     switch (statusForRound) {
       case ACTIVE -> outcomeDescription = selectedCandidate;
-      case INACTIVE_BY_UNDERVOTE -> outcomeDescription = "undervote" + additionalLogText;
-      case INACTIVE_BY_OVERVOTE -> outcomeDescription = "overvote" + additionalLogText;
-      case INACTIVE_BY_SKIPPED_RANKING -> outcomeDescription =
-          "exhausted by skipped ranking" + additionalLogText;
-      case INACTIVE_BY_REPEATED_RANKING -> outcomeDescription =
-          "duplicate candidate" + additionalLogText;
-      case INACTIVE_BY_EXHAUSTED_CHOICES -> outcomeDescription =
-          "no continuing candidate" + additionalLogText;
+      case DID_NOT_RANK_ANY_CANDIDATES -> outcomeDescription =
+          "did not rank any candidates" + additionalLogText;
+      case INVALIDATED_BY_OVERVOTE -> outcomeDescription =
+          "invalidated by overvote" + additionalLogText;
+      case EXHAUSTED_CHOICE -> outcomeDescription =
+          "exhausted choice" + additionalLogText;
+      case INVALIDATED_BY_SKIPPED_RANKING -> outcomeDescription =
+          "invalidated by skipped ranking" + additionalLogText;
+      case INVALIDATED_BY_REPEATED_RANKING -> outcomeDescription =
+          "invalidated by repeated ranking" + additionalLogText;
       default ->
       // Programming error: we missed a status here
       throw new RuntimeException("Unexpected ballot status: " + statusForRound);
@@ -1068,7 +1070,7 @@ && isCandidateContinuing(cvr.getCurrentRecipientOfVote())) {
       // check for a CVR with no rankings at all
       if (cvr.candidateRankings.numRankings() == 0) {
         recordSelectionForCastVoteRecord(
-            cvr, roundTally, null, StatusForRound.INACTIVE_BY_UNDERVOTE, "");
+            cvr, roundTally, null, StatusForRound.DID_NOT_RANK_ANY_CANDIDATES, "");
       }
 
       // iterate through the rankings in this cvr from most to least preferred.
@@ -1097,7 +1099,7 @@ && isCandidateContinuing(cvr.getCurrentRecipientOfVote())) {
         if (config.getMaxSkippedRanksAllowed() != Integer.MAX_VALUE
             && (rank - lastRankSeen > config.getMaxSkippedRanksAllowed() + 1)) {
           recordSelectionForCastVoteRecord(
-              cvr, roundTally, null, StatusForRound.INACTIVE_BY_SKIPPED_RANKING, "");
+              cvr, roundTally, null, StatusForRound.INVALIDATED_BY_SKIPPED_RANKING, "");
           break;
         }
         lastRankSeen = rank;
@@ -1118,7 +1120,7 @@ && isCandidateContinuing(cvr.getCurrentRecipientOfVote())) {
                 cvr,
                 roundTally,
                 null,
-                StatusForRound.INACTIVE_BY_REPEATED_RANKING,
+                StatusForRound.INVALIDATED_BY_REPEATED_RANKING,
                 " " + duplicateCandidate);
             break;
           }
@@ -1128,14 +1130,14 @@ && isCandidateContinuing(cvr.getCurrentRecipientOfVote())) {
         OvervoteDecision overvoteDecision = getOvervoteDecision(candidates);
         if (overvoteDecision == OvervoteDecision.EXHAUST) {
           recordSelectionForCastVoteRecord(
-              cvr, roundTally, null, StatusForRound.INACTIVE_BY_OVERVOTE, "");
+              cvr, roundTally, null, StatusForRound.INVALIDATED_BY_OVERVOTE, "");
           break;
         } else if (overvoteDecision == OvervoteDecision.SKIP_TO_NEXT_RANK) {
           if (rank == cvr.candidateRankings.maxRankingNumber()) {
             // If the final ranking is an overvote, even if we're trying to skip to the next rank,
             // we consider this inactive by exhausted choices -- not an overvote.
             recordSelectionForCastVoteRecord(
-                cvr, roundTally, null, StatusForRound.INACTIVE_BY_EXHAUSTED_CHOICES, "");
+                cvr, roundTally, null, StatusForRound.EXHAUSTED_CHOICE, "");
           }
           continue;
         }
@@ -1170,22 +1172,9 @@ && isCandidateContinuing(cvr.getCurrentRecipientOfVote())) {
         }
 
         // if this is the last ranking we are out of rankings and must exhaust this cvr
-        // determine if the reason is skipping too many ranks, or no continuing candidates
         if (rank == cvr.candidateRankings.maxRankingNumber()) {
-          // When determining if this is an undervote or exhausted choice, look at either
-          // the max ranking allowed by the config, or if the config does not impose a limit,
-          // look at the number of declared candidates.
-          int maxAllowedRanking = config.isMaxRankingsSetToMaximum()
-                  ? config.getNumDeclaredCandidates()
-                  : config.getMaxRankingsAllowedWhenNotSetToMaximum();
-          if (config.getMaxSkippedRanksAllowed() != Integer.MAX_VALUE
-              && maxAllowedRanking - rank > config.getMaxSkippedRanksAllowed()) {
-            recordSelectionForCastVoteRecord(
-                cvr, roundTally, null, StatusForRound.INACTIVE_BY_UNDERVOTE, "");
-          } else {
-            recordSelectionForCastVoteRecord(
-                cvr, roundTally, null, StatusForRound.INACTIVE_BY_EXHAUSTED_CHOICES, "");
-          }
+          recordSelectionForCastVoteRecord(
+              cvr, roundTally, null, StatusForRound.EXHAUSTED_CHOICE, "");
         }
       } // end looping over the rankings within one ballot
     } // end looping over all ballots

src/test/resources/network/brightspots/rcv/test_data/2013_minneapolis_mayor/2013_minneapolis_mayor_expected_MINNEAPOLIS_W-13_P-09_precinct_summary.csv

@@ -14,7 +14,7 @@ Contest Summary
 Number to be Elected,1
 Number of Candidates,36
 Total Number of Ballots,985
-Number of Undervotes,0
+Number of Undervotes (No Rankings),0
 
 Rounds,Round 1 Votes,% of vote,transfer,Round 2 Votes,% of vote,transfer,Round 3 Votes,% of vote,transfer,Round 4 Votes,% of vote,transfer,Round 5 Votes,% of vote,transfer,Round 6 Votes,% of vote,transfer,Round 7 Votes,% of vote,transfer,Round 8 Votes,% of vote,transfer,Round 9 Votes,% of vote,transfer,Round 10 Votes,% of vote,transfer,Round 11 Votes,% of vote,transfer,Round 12 Votes,% of vote,transfer,Round 13 Votes,% of vote,transfer,Round 14 Votes,% of vote,transfer,Round 15 Votes,% of vote,transfer,Round 16 Votes,% of vote,transfer,Round 17 Votes,% of vote,transfer,Round 18 Votes,% of vote,transfer,Round 19 Votes,% of vote,transfer,Round 20 Votes,% of vote,transfer,Round 21 Votes,% of vote,transfer,Round 22 Votes,% of vote,transfer,Round 23 Votes,% of vote,transfer,Round 24 Votes,% of vote,transfer,Round 25 Votes,% of vote,transfer,Round 26 Votes,% of vote,transfer,Round 27 Votes,% of vote,transfer,Round 28 Votes,% of vote,transfer,Round 29 Votes,% of vote,transfer,Round 30 Votes,% of vote,transfer,Round 31 Votes,% of vote,transfer,Round 32 Votes,% of vote,transfer,Round 33 Votes,% of vote,transfer
 BETSY HODGES,359,36.44%,0,359,36.44%,0,359,36.44%,0,359,36.48%,0,359,36.48%,0,359,36.48%,0,359,36.48%,0,359,36.48%,0,359,36.48%,0,359,36.52%,0,359,36.52%,0,359,36.59%,0,359,36.59%,0,359,36.59%,0,359,36.59%,0,359,36.59%,0,359,36.59%,0,359,36.59%,0,359,36.59%,0,359,36.63%,2,361,36.87%,0,361,36.87%,0,361,36.87%,0,361,36.87%,0,361,36.87%,0,361,36.87%,0,361,37.13%,0,361,37.13%,1,362,37.35%,2,364,37.6%,2,366,38.4%,23,389,41.33%,69,458,57.39%,0