Improvements to Inactive Ballot tracking, and corrected reporting of undervotes

src/main/java/network/brightspots/rcv/BaseCvrReader.java

@@ -119,6 +119,15 @@ Map<String, Integer> gatherUnknownCandidates(
     return unrecognizedCandidateCounts;
   }
 
+  boolean usesLastAllowedRanking(List<Pair<Integer, String>> rankings, String contestId) {
+    if (rankings.isEmpty()) {
+      return false;
+    }
+
+    int lastRanking = rankings.get(rankings.size() - 1).getKey();
+    return !isRankingAllowed(lastRanking + 1, contestId);
+  }
+
   // Human-readable name for output logs
   public abstract String readerName();
 }

src/main/java/network/brightspots/rcv/CastVoteRecord.java

@@ -36,6 +36,8 @@ class CastVoteRecord {
   private final String precinct;
   // which precinct portion this ballot came from
   private final String precinctPortion;
+  // is the last-used ranking the last-allowed ranking in the CVR?
+  private final boolean usesLastAllowedRanking;
   // records winners to whom some fraction of this vote has been allocated
   private final Map<String, BigDecimal> winnerToFractionalValue = new HashMap<>();
   // If CVR CDF output is enabled, we store the necessary info here: for each round, the list of
@@ -65,8 +67,10 @@ class CastVoteRecord {
       String suppliedId,
       String precinct,
       String precinctPortion,
+      boolean usesLastAllowedRanking,
       List<Pair<Integer, String>> rankings) {
-    this(contestId, tabulatorId, batchId, suppliedId, null, precinct, precinctPortion, rankings);
+    this(contestId, tabulatorId, batchId, suppliedId, null, precinct, precinctPortion,
+        usesLastAllowedRanking, rankings);
   }
 
   CastVoteRecord(
@@ -77,6 +81,7 @@ class CastVoteRecord {
           String computedId,
           String precinct,
           String precinctPortion,
+          boolean usesLastAllowedRanking,
           List<Pair<Integer, String>> rankings) {
     this.contestId = contestId;
     this.tabulatorId = tabulatorId;
@@ -85,6 +90,7 @@ class CastVoteRecord {
     this.computedId = computedId;
     this.precinct = precinct;
     this.precinctPortion = precinctPortion;
+    this.usesLastAllowedRanking = usesLastAllowedRanking;
     this.candidateRankings = new CandidateRankingsList(rankings);
   }
 
@@ -93,8 +99,10 @@ class CastVoteRecord {
       String suppliedId,
       String precinct,
       String batchId,
+      boolean usesLastAllowedRanking,
       List<Pair<Integer, String>> rankings) {
-    this(null, null, batchId, suppliedId, computedId, precinct, null, rankings);
+    this(null, null, batchId, suppliedId, computedId, precinct, null,
+        usesLastAllowedRanking, rankings);
   }
 
   String getContestId() {
@@ -116,6 +124,10 @@ String getPrecinctPortion() {
     return precinctPortion;
   }
 
+  boolean doesUseLastAllowedRanking() {
+    return usesLastAllowedRanking;
+  }
+
   String getId() {
     return suppliedId != null ? suppliedId : computedId;
   }
@@ -224,11 +236,11 @@ Map<String, BigDecimal> getWinnerToFractionalValue() {
   // it is active or not.
   enum StatusForRound {
     ACTIVE,
-    INACTIVE_BY_UNDERVOTE,
-    INACTIVE_BY_OVERVOTE,
-    INACTIVE_BY_SKIPPED_RANKING,
-    INACTIVE_BY_REPEATED_RANKING,
-    INACTIVE_BY_EXHAUSTED_CHOICES
+    DID_NOT_RANK_ANY_CANDIDATES,
+    EXHAUSTED_CHOICE,
+    INVALIDATED_BY_OVERVOTE,
+    INVALIDATED_BY_SKIPPED_RANKING,
+    INVALIDATED_BY_REPEATED_RANKING,
   }
 
   enum VoteOutcomeType {

src/main/java/network/brightspots/rcv/ClearBallotCvrReader.java

@@ -110,6 +110,7 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
                 cvrData[CvrColumnField.BallotID.ordinal()],
                 cvrData[CvrColumnField.PrecinctID.ordinal()],
                 null,
+                usesLastAllowedRanking(rankings, null),
                 rankings);
 
         castVoteRecords.add(castVoteRecord);

src/main/java/network/brightspots/rcv/CommonDataFormatReader.java

@@ -316,7 +316,12 @@ void parseXml(List<CastVoteRecord> castVoteRecords) throws CvrParseException, IO
         String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
         // create the new CastVoteRecord
         CastVoteRecord newRecord = new CastVoteRecord(
-            computedCastVoteRecordId, cvr.UniqueId, precinctId, cvr.BatchSequenceId, rankings);
+            computedCastVoteRecordId,
+            cvr.UniqueId,
+            precinctId,
+            cvr.BatchSequenceId,
+            usesLastAllowedRanking(rankings, null),
+            rankings);
         castVoteRecords.add(newRecord);
 
         // provide some user feedback on the CVR count
@@ -527,8 +532,13 @@ void parseJson(List<CastVoteRecord> castVoteRecords) throws CvrParseException {
       String batchId = (String) cvr.get("BatchSequenceId");
       String computedCastVoteRecordId = String.format("%s(%d)", fileName, ++cvrIndex);
       // create the new CastVoteRecord
-      CastVoteRecord newRecord =
-          new CastVoteRecord(computedCastVoteRecordId, ballotId, precinctId, batchId, rankings);
+      CastVoteRecord newRecord = new CastVoteRecord(
+          computedCastVoteRecordId,
+          ballotId,
+          precinctId,
+          batchId,
+          usesLastAllowedRanking(rankings, null),
+          rankings);
       castVoteRecords.add(newRecord);
       // provide some user feedback on the CVR count
       if (castVoteRecords.size() % 50000 == 0) {

src/main/java/network/brightspots/rcv/CsvCvrReader.java

@@ -99,9 +99,13 @@ void readCastVoteRecords(List<CastVoteRecord> castVoteRecords)
         }
 
         // create the new CastVoteRecord
-        CastVoteRecord newCvr =
-            new CastVoteRecord(
-                Integer.toString(index), "no supplied ID", "no precinct", "no batch ID", rankings);
+        CastVoteRecord newCvr = new CastVoteRecord(
+            Integer.toString(index),
+            "no supplied ID",
+            "no precinct",
+            "no batch ID",
+            usesLastAllowedRanking(rankings, null),
+            rankings);
         castVoteRecords.add(newCvr);
       }
     } catch (IOException exception) {

src/main/java/network/brightspots/rcv/DominionCvrReader.java

@@ -357,9 +357,16 @@ private int parseCvrFile(
             rankings.add(ranking);
           }
           // create the new cvr
-          CastVoteRecord newCvr =
-              new CastVoteRecord(contestId, tabulatorId, batchId, suppliedId,
-                      computedId, precinct, precinctPortion, rankings);
+          CastVoteRecord newCvr = new CastVoteRecord(
+              contestId,
+              tabulatorId,
+              batchId,
+              suppliedId,
+              computedId,
+              precinct,
+              precinctPortion,
+              usesLastAllowedRanking(rankings, contestId),
+              rankings);
           castVoteRecords.add(newCvr);
         }
       }

src/main/java/network/brightspots/rcv/GuiTabulateController.java

@@ -274,6 +274,10 @@ private <T> void watchGenericService(
           originalCallback.handle(workerStateEvent);
           onSuccessCallback.handle(workerStateEvent);
         });
+    service.setOnFailed(
+        workerStateEvent -> {
+          stage.setOnCloseRequest(null);
+        });
   }
 
   /**

src/main/java/network/brightspots/rcv/HartCvrReader.java

@@ -135,6 +135,7 @@ private void readCastVoteRecord(List<CastVoteRecord> castVoteRecords, Path path)
                 xmlCvr.CvrGuid,
                 xmlCvr.PrecinctSplit.Name,
                 xmlCvr.PrecinctSplit.Id,
+                usesLastAllowedRanking(rankings, null),
                 rankings);
         castVoteRecords.add(cvr);
 

src/main/java/network/brightspots/rcv/ResultsWriter.java

@@ -374,18 +374,21 @@ private void generateSummarySpreadsheet(
     }
     csvPrinter.println();
 
-    Pair<String, StatusForRound>[] statusesToPrint =
-        new Pair[] {
-          new Pair<>("Overvotes", StatusForRound.INACTIVE_BY_OVERVOTE),
-          new Pair<>("Skipped Rankings", StatusForRound.INACTIVE_BY_SKIPPED_RANKING),
-          new Pair<>("Exhausted Choices", StatusForRound.INACTIVE_BY_EXHAUSTED_CHOICES),
-          new Pair<>("Repeated Rankings", StatusForRound.INACTIVE_BY_REPEATED_RANKING)
-        };
+    List<Pair<String, StatusForRound>> statusesToPrint = new ArrayList<>();
+    statusesToPrint.add(new Pair<>("Overvotes",
+            StatusForRound.INVALIDATED_BY_OVERVOTE));
+    statusesToPrint.add(new Pair<>("Skipped Rankings",
+            StatusForRound.INVALIDATED_BY_SKIPPED_RANKING));
+    statusesToPrint.add(new Pair<>("Exhausted Choices",
+            StatusForRound.EXHAUSTED_CHOICE));
+    statusesToPrint.add(new Pair<>("Repeated Rankings",
+            StatusForRound.INVALIDATED_BY_REPEATED_RANKING));
 
     for (Pair<String, StatusForRound> statusToPrint : statusesToPrint) {
       csvPrinter.print("Inactive Ballots by " + statusToPrint.getKey());
+
+      StatusForRound status = statusToPrint.getValue();
       for (int round = 1; round <= numRounds; round++) {
-        StatusForRound status = statusToPrint.getValue();
         BigDecimal thisRoundInactive = roundTallies.get(round).getBallotStatusTally(status);
         csvPrinter.print(thisRoundInactive);
 
@@ -405,14 +408,11 @@ private void generateSummarySpreadsheet(
     }
 
     csvPrinter.print("Inactive Ballots Total");
-    // Undervotes should not be included in the Inactive Ballots count, even though we treat them
-    // as such internally. Subtract undervotes (which are static throughout a contest) from the
-    // inactive ballot totals.
-    BigDecimal numUndervotes =
-        roundTallies.get(1).getBallotStatusTally(StatusForRound.INACTIVE_BY_UNDERVOTE);
+    BigDecimal numNoRankings =
+        roundTallies.get(1).getBallotStatusTally(StatusForRound.DID_NOT_RANK_ANY_CANDIDATES);
     for (int round = 1; round <= numRounds; round++) {
       BigDecimal thisRoundInactive = roundTallies.get(round).inactiveBallotSum();
-      csvPrinter.print(thisRoundInactive.subtract(numUndervotes));
+      csvPrinter.print(thisRoundInactive.subtract(numNoRankings));
 
       // Don't display percentage of inactive ballots
       csvPrinter.print("");
@@ -469,15 +469,15 @@ private void addActionRows(CSVPrinter csvPrinter) throws IOException {
 
   private void addContestSummaryRows(CSVPrinter csvPrinter, RoundTally round1Tally)
       throws IOException {
-    BigDecimal numUndervotes =
-        round1Tally.getBallotStatusTally(StatusForRound.INACTIVE_BY_UNDERVOTE);
+    BigDecimal numNoRankings =
+        round1Tally.getBallotStatusTally(StatusForRound.DID_NOT_RANK_ANY_CANDIDATES);
     BigDecimal totalNumberBallots =
         round1Tally.activeBallotSum().add(round1Tally.inactiveBallotSum());
     csvPrinter.printRecord("Contest Summary");
     csvPrinter.printRecord("Number to be Elected", config.getNumberOfWinners());
     csvPrinter.printRecord("Number of Candidates", config.getNumCandidates());
     csvPrinter.printRecord("Total Number of Ballots", totalNumberBallots);
-    csvPrinter.printRecord("Number of Undervotes", numUndervotes);
+    csvPrinter.printRecord("Number of Undervotes (No Rankings)", numNoRankings);
     csvPrinter.println();
   }
 
@@ -939,8 +939,8 @@ private void generateSummaryJson(
       configData.put(slice.toLowerString(), sliceId);
     }
 
-    BigDecimal firstRoundUndervotes =
-        roundTallies.get(1).getBallotStatusTally(StatusForRound.INACTIVE_BY_UNDERVOTE);
+    BigDecimal numNoRankings =
+        roundTallies.get(1).getBallotStatusTally(StatusForRound.DID_NOT_RANK_ANY_CANDIDATES);
     BigDecimal totalNumberBallots =
         roundTallies.get(1).activeBallotSum().add(roundTallies.get(1).inactiveBallotSum());
     BigDecimal lastRoundThreshold = roundTallies.get(numRounds).getWinningThreshold();
@@ -950,7 +950,7 @@ private void generateSummaryJson(
     summaryData.put("numWinners", config.getNumberOfWinners());
     summaryData.put("numCandidates", config.getCandidateNames().size());
     summaryData.put("totalNumBallots", totalNumberBallots);
-    summaryData.put("undervotes", firstRoundUndervotes.toBigInteger());
+    summaryData.put("undervotes", numNoRankings.toBigInteger());
 
     ArrayList<Object> results = new ArrayList<>();
     for (int round = 1; round <= numRounds; round++) {
@@ -992,10 +992,14 @@ private Map<String, BigDecimal> getInactiveJsonMap(RoundTally roundTally) {
     Map<String, BigDecimal> inactiveMap = new HashMap<>();
     Pair<String, StatusForRound>[] statusesToPrint =
         new Pair[] {
-          new Pair<>("overvotes", StatusForRound.INACTIVE_BY_OVERVOTE),
-          new Pair<>("skippedRankings", StatusForRound.INACTIVE_BY_SKIPPED_RANKING),
-          new Pair<>("exhaustedChoices", StatusForRound.INACTIVE_BY_EXHAUSTED_CHOICES),
-          new Pair<>("repeatedRankings", StatusForRound.INACTIVE_BY_REPEATED_RANKING)
+          new Pair<>("overvotes",
+                     StatusForRound.INVALIDATED_BY_OVERVOTE),
+          new Pair<>("skippedRankings",
+                     StatusForRound.INVALIDATED_BY_SKIPPED_RANKING),
+          new Pair<>("repeatedRankings",
+                     StatusForRound.INVALIDATED_BY_REPEATED_RANKING),
+          new Pair<>("exhaustedChoices",
+                     StatusForRound.EXHAUSTED_CHOICE),
         };
     for (Pair<String, StatusForRound> statusToPrint : statusesToPrint) {
       inactiveMap.put(

src/main/java/network/brightspots/rcv/StreamingCvrReader.java

@@ -221,6 +221,7 @@ private void endCvr() {
         currentSuppliedCvrId,
         currentPrecinct,
         currentBatch,
+        usesLastAllowedRanking(currentRankings, null),
         currentRankings);
     cvrList.add(newRecord);