Implement bulk uploading ballots from the UI

packages/backend/sample.env

@@ -14,8 +14,8 @@ ALLOWED_URLS='http://localhost:3000' # 3000 should match FRONTEND_PORT from fron
 BACKEND_PORT=5000 # if updated, make sure to also change the proxy and socket urls in the frontend .env
 
 #### FRONT PAGE STATS ####
-CLASSIC_ELECTION_COUNT=500
-CLASSIC_VOTE_COUNT=5000
+CLASSIC_ELECTION_COUNT=0
+CLASSIC_VOTE_COUNT=0
 
 #### EMAIL ####
 # Contact [email protected] if you need access for developing email features

packages/backend/src/Controllers/Ballot/castVoteController.ts

@@ -15,6 +15,8 @@ import { IElectionRequest } from "../../IRequest";
 import { Response, NextFunction } from 'express';
 import { io } from "../../socketHandler";
 import { Server } from "socket.io";
+import { expectPermission } from "../controllerUtils";
+import { permissions } from "@equal-vote/star-vote-shared/domain_model/permissions";
 
 const ElectionsModel = ServiceLocator.electionsDb();
 const ElectionRollModel = ServiceLocator.electionRollDb();
@@ -29,9 +31,12 @@ type CastVoteEvent = {
     userEmail?:string,
 }
 
+// NOTE: discord isn't implemented yet, but that's the plan for the future
+type BallotSubmitType = 'submitted_via_browser' | 'submitted_via_admin' | 'submitted_via_discord';
+
 const castVoteEventQueue = "castVoteEvent";
 
-async function makeBallotEvent(req: IElectionRequest, targetElection: Election, inputBallot: Ballot, voter_id?: string){
+async function makeBallotEvent(req: IElectionRequest, targetElection: Election, inputBallot: Ballot, submitType: BallotSubmitType, voter_id?: string){
     inputBallot.election_id = targetElection.election_id;
     let roll = null;
 
@@ -44,8 +49,10 @@ async function makeBallotEvent(req: IElectionRequest, targetElection: Election,
             throw new Unauthorized(missingAuthData);
         }
 
-        roll = await getOrCreateElectionRoll(req, targetElection, req);
+        // skipping state check since this is allowed when uploading ballots, and it's already explicitly checked for individual ballots
+        roll = await getOrCreateElectionRoll(req, targetElection, req, voter_id, true);
         const voterAuthorization = getVoterAuthorization(roll,missingAuthData)
+
         assertVoterMayVote(voterAuthorization, req);
 
         //TODO: currently we have both a value on the input Ballot, and the route param.
@@ -70,7 +77,7 @@ async function makeBallotEvent(req: IElectionRequest, targetElection: Election,
     //TODO, ensure the user ID is added to the ballot...
     //should server-authenticate the user id based on auth token
     inputBallot.history.push({
-        action_type:"submit",
+        action_type: submitType,
         actor: roll===null ? '' : roll.voter_id ,
         timestamp:inputBallot.date_submitted,
     });
@@ -101,6 +108,10 @@ async function makeBallotEvent(req: IElectionRequest, targetElection: Election,
 async function uploadBallotsController(req: IElectionRequest, res: Response, next: NextFunction) {
     Logger.info(req, "Upload Ballots Controller");
 
+    expectPermission(req.user_auth.roles, permissions.canUploadBallots);
+
+    //TODO: if it's a public_archive item, also check canUpdatePublicArchive instead
+
     const targetElection = req.election;
     if (targetElection == null){
         const errMsg = "Invalid Ballot: invalid election Id";
@@ -110,7 +121,7 @@ async function uploadBallotsController(req: IElectionRequest, res: Response, nex
 
     let events = await Promise.all(
         req.body.ballots.map(({ballot, voter_id} : {ballot: Ballot, voter_id: string}) => 
-            makeBallotEvent(req, targetElection, structuredClone(ballot), voter_id).catch((err) => ({
+            makeBallotEvent(req, targetElection, structuredClone(ballot), 'submitted_via_admin', voter_id).catch((err) => ({
                 error: err,
                 ballot: ballot
             }))
@@ -154,7 +165,7 @@ async function castVoteController(req: IElectionRequest, res: Response, next: Ne
         throw new BadRequest("Election is not open");
     }
 
-    let event = await makeBallotEvent(req, targetElection, req.body.ballot)
+    let event = await makeBallotEvent(req, targetElection, req.body.ballot, 'submitted_via_browser')
 
     event.userEmail = req.body.receiptEmail;
 

packages/backend/src/Controllers/Election/getElectionsController.ts

@@ -9,6 +9,7 @@ import { Election, removeHiddenFields } from '@equal-vote/star-vote-shared/domai
 var ElectionsModel = ServiceLocator.electionsDb();
 var ElectionRollModel = ServiceLocator.electionRollDb();
 
+// TODO: We should probably split this up as the user will only need one of these filters
 const getElections = async (req: IElectionRequest, res: Response, next: NextFunction) => {
     Logger.info(req, `getElections`);
     // var filter = (req.query.filter == undefined) ? "" : req.query.filter;
@@ -51,14 +52,12 @@ const getElections = async (req: IElectionRequest, res: Response, next: NextFunc
         }
     }
 
-    /////////// OPEN ELECTIONS ////////////////
-    var open_elections = await ElectionsModel.getOpenElections(req);
-
     res.json({
         elections_as_official,
         elections_as_unsubmitted_voter,
         elections_as_submitted_voter,
-        open_elections
+        public_archive_elections: await ElectionsModel.getPublicArchiveElections(req),
+        open_elections: await ElectionsModel.getOpenElections(req)
     });
 }
 
@@ -67,18 +66,25 @@ const innerGetGlobalElectionStats = async (req: IRequest) => {
 
     let electionVotes = await ElectionsModel.getBallotCountsForAllElections(req);
 
+    let sourcedFromPrior = await ElectionsModel.getElectionsSourcedFromPrior(req);
+    let priorElections = sourcedFromPrior?.map(e => e.election_id) ?? [];
+
     let stats = {
         elections: Number(process.env.CLASSIC_ELECTION_COUNT ?? 0),
         votes: Number(process.env.CLASSIC_VOTE_COUNT ?? 0),
     };
 
-    electionVotes?.map(m => m['v'])?.forEach((count) => {
-        stats['votes'] = stats['votes'] + Number(count);
-        if(count >= 2){
-            stats['elections'] = stats['elections'] + 1;
+    electionVotes
+        ?.filter(m => !priorElections.includes(m['election_id']))
+        ?.map(m => m['v'])
+        ?.forEach((count) => {
+            stats['votes'] = stats['votes'] + Number(count);
+            if(count >= 2){
+                stats['elections'] = stats['elections'] + 1;
+            }
+            return stats;
         }
-        return stats;
-    });
+    );
 
     return stats;
 }

packages/backend/src/Controllers/Roll/voterRollUtils.ts

@@ -10,7 +10,7 @@ import { hashString } from "../controllerUtils";
 
 const ElectionRollModel = ServiceLocator.electionRollDb();
 
-export async function getOrCreateElectionRoll(req: IRequest, election: Election, ctx: ILoggingContext): Promise<ElectionRoll | null> {
+export async function getOrCreateElectionRoll(req: IRequest, election: Election, ctx: ILoggingContext, voter_id_override?: string, skipStateCheck?: boolean): Promise<ElectionRoll | null> {
     // Checks for existing election roll for user
     Logger.info(req, `getOrCreateElectionRoll`)
     const ip_hash = hashString(req.ip!)
@@ -23,9 +23,9 @@ export async function getOrCreateElectionRoll(req: IRequest, election: Election,
     if (election.settings.voter_authentication.voter_id && election.settings.voter_access == 'closed') {
         // cookies don't support special charaters
         // https://help.vtex.com/en/tutorial/why-dont-cookies-support-special-characters--6hs7MQzTri6Yg2kQoSICoQ
-        voter_id = atob(req.cookies?.voter_id); 
+        voter_id = voter_id_override ?? atob(req.cookies?.voter_id); 
     } else if (election.settings.voter_authentication.voter_id && election.settings.voter_access == 'open') {
-        voter_id = req.user?.sub
+        voter_id = voter_id_override ?? req.user?.sub
     }
 
     // Get all election roll entries that match any of the voter authentication fields
@@ -38,12 +38,9 @@ export async function getOrCreateElectionRoll(req: IRequest, election: Election,
 
     if (electionRollEntries == null) {
         // No election roll found, create one if voter access is open and election state is open
-        if (election.settings.voter_access !== 'open') {
-            return null
-        }
-        if (election.state !== 'open') {
-            return null
-        }
+        if (election.settings.voter_access !== 'open') return null
+        if (!skipStateCheck && election.state !== 'open') return null
+
         Logger.info(req, "Creating new roll");
         const new_voter_id = election.settings.voter_authentication.voter_id ? voter_id : randomUUID()
         const history = [{

packages/backend/src/Migrations/2025_01_29_admin_upload.ts

@@ -0,0 +1,31 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<any>): Promise<void> {
+    await db.schema.alterTable('electionDB')
+        /* ballot_source types
+        live_election: Ballots submitted by voters during election
+        prior_election: Election admin uploaded ballots from a previous election
+        */
+        .addColumn('ballot_source', 'varchar' )
+        // unique identifier for mapping public archive elections to their real elections
+        // ex. Genola_11022021_CityCouncil
+        .addColumn('public_archive_id', 'varchar' )
+        // support_email is obsolete, it has been superceded by settings.contact_email
+        .dropColumn('support_email')
+        .execute()
+
+    await db.updateTable('electionDB')
+        .set({
+            ballot_source: 'live_election',
+            public_archive_id: null,
+        })
+        .execute()
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+    await db.schema.alterTable('electionDB')
+        .dropColumn('ballot_source')
+        .dropColumn('public_archive_id')
+        .addColumn('support_email', 'varchar')
+        .execute()
+}

packages/backend/src/Models/Elections.ts

@@ -11,6 +11,7 @@ import { InternalServerError } from '@curveball/http-errors';
 const tableName = 'electionDB';
 
 interface IVoteCount{
+    election_id: string;
     v: number;
 }
 
@@ -95,6 +96,19 @@ export default class ElectionsDB implements IElectionStore {
         });
     }
 
+    async getPublicArchiveElections(ctx: ILoggingContext): Promise<Election[] | null> {
+        Logger.debug(ctx, `${tableName}.getPublicArchiveElections`);
+        // Returns all elections where settings.voter_access == open and state == open
+
+        // TODO: The filter is pretty inefficient for now since I don't think there's a way to include on settings.voter_access in the query
+        return await this._postgresClient
+            .selectFrom(tableName)
+            .where('head', '=', true)
+            .where('public_archive_id', '!=', null)
+            .selectAll()
+            .execute()
+    }
+
     getElections(id: string, email: string, ctx: ILoggingContext): Promise<Election[] | null> {
         // When I filter in trello it adds "filter=member:arendpetercastelein,overdue:true" to the URL, I'm following the same pattern here
         Logger.debug(ctx, `${tableName}.getAll ${id}`);
@@ -113,13 +127,23 @@ export default class ElectionsDB implements IElectionStore {
             )
         }
 
-
-        const elections = query.execute().catch(dneCatcher)
+        return query.execute().catch(dneCatcher)
+    }
 
-        return elections
+    getElectionsSourcedFromPrior(ctx: ILoggingContext): Promise<Election[] | null> {
+        // When I filter in trello it adds "filter=member:arendpetercastelein,overdue:true" to the URL, I'm following the same pattern here
+        Logger.debug(ctx, `${tableName}.getSourcedFromPrior`);
+
+        return this._postgresClient
+            .selectFrom(tableName)
+            .where('ballot_source', '=', 'prior_election')
+            .where('head', '=', true)
+            .selectAll()
+            .execute()
+            .catch(dneCatcher);
     }
 
-    // TODO: I'm a bit lazy for now just having Object as the type
+    // TODO: this function should probably be in the ballots model
     getBallotCountsForAllElections(ctx: ILoggingContext): Promise<IVoteCount[] | null> {
         Logger.debug(ctx, `${tableName}.getAllElectionsWithBallotCounts`);
 
@@ -129,6 +153,7 @@ export default class ElectionsDB implements IElectionStore {
             .select(
                 (eb) => eb.fn.count('ballot_id').as('v')
             )
+            .select('election_id')
             .where('head', '=', true)
             .groupBy('election_id')
             .orderBy('election_id')