diff --git a/.gitignore b/.gitignore index 79d8a36f..d9de6d2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# vscode +.vscode + # Logs logs *.log diff --git a/.prettierrc b/.prettierrc index 27f4fbb9..153277ad 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,5 +1,5 @@ { - "printWidth": 120, + "printWidth": 140, "tabWidth": 4, "semi": true, "endOfLine": "auto" diff --git a/databases/_private.db.sql b/databases/_private.db.sql index a3c413d2..4a1f8038 100644 --- a/databases/_private.db.sql +++ b/databases/_private.db.sql @@ -26,24 +26,6 @@ CREATE TABLE IF NOT EXISTS "config" ( "value" TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS "titleVotes" ( - "id" SERIAL PRIMARY KEY, - "videoID" TEXT NOT NULL, - "UUID" TEXT NOT NULL, - "userID" TEXT NOT NULL, - "hashedIP" TEXT NOT NULL, - "type" INTEGER NOT NULL -); - -CREATE TABLE IF NOT EXISTS "thumbnailVotes" ( - "id" SERIAL PRIMARY KEY, - "videoID" TEXT NOT NULL, - "UUID" TEXT NOT NULL, - "userID" TEXT NOT NULL, - "hashedIP" TEXT NOT NULL, - "type" INTEGER NOT NULL -); - CREATE TABLE IF NOT EXISTS "portVideo" ( "bvID" TEXT NOT NULL, "UUID" TEXT PRIMARY KEY, diff --git a/databases/_private_indexes.sql b/databases/_private_indexes.sql index 647fe616..af628561 100644 --- a/databases/_private_indexes.sql +++ b/databases/_private_indexes.sql @@ -2,7 +2,7 @@ CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_v4" ON public."sponsorTimes" USING btree - ("videoID" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST); + ("videoID" ASC NULLS LAST, "cid" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST); -- votes diff --git a/databases/_sponsorTimes.db.sql b/databases/_sponsorTimes.db.sql index 9bc79d3c..f53afbfa 100644 --- a/databases/_sponsorTimes.db.sql +++ b/databases/_sponsorTimes.db.sql @@ -44,49 +44,6 @@ CREATE TABLE IF NOT EXISTS "config" ( "value" TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS "titles" ( - "videoID" TEXT NOT NULL, - "title" TEXT NOT NULL, - "original" INTEGER default 0, - "userID" TEXT NOT NULL, - "service" TEXT NOT NULL, - "hashedVideoID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "UUID" TEXT NOT NULL PRIMARY KEY -); - -CREATE TABLE IF NOT EXISTS "titleVotes" ( - "UUID" TEXT NOT NULL PRIMARY KEY, - "votes" INTEGER NOT NULL default 0, - "locked" INTEGER NOT NULL default 0, - "shadowHidden" INTEGER NOT NULL default 0, - FOREIGN KEY("UUID") REFERENCES "titles"("UUID") -); - -CREATE TABLE IF NOT EXISTS "thumbnails" ( - "videoID" TEXT NOT NULL, - "original" INTEGER default 0, - "userID" TEXT NOT NULL, - "service" TEXT NOT NULL, - "hashedVideoID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "UUID" TEXT NOT NULL PRIMARY KEY -); - -CREATE TABLE IF NOT EXISTS "thumbnailTimestamps" ( - "UUID" TEXT NOT NULL PRIMARY KEY, - "timestamp" INTEGER NOT NULL default 0, - FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID") -); - -CREATE TABLE IF NOT EXISTS "thumbnailVotes" ( - "UUID" TEXT NOT NULL PRIMARY KEY, - "votes" INTEGER NOT NULL default 0, - "locked" INTEGER NOT NULL default 0, - "shadowHidden" INTEGER NOT NULL default 0, - FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID") -); - CREATE TABLE IF NOT EXISTS "portVideo" ( "bvID" TEXT NOT NULL, "ytbID" TEXT NOT NULL, diff --git a/databases/_upgrade_private_14.sql b/databases/_upgrade_private_14.sql new file mode 100644 index 00000000..3d640185 --- /dev/null +++ b/databases/_upgrade_private_14.sql @@ -0,0 +1,7 @@ +BEGIN TRANSACTION; + +ALTER TABLE "sponsorTimes" ADD "cid" TEXT NOT NULL DEFAULT ''; + +UPDATE "config" SET value = 14 WHERE key = 'version'; + +COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_private_4.sql b/databases/_upgrade_private_4.sql index 008cb673..d378dc15 100644 --- a/databases/_upgrade_private_4.sql +++ b/databases/_upgrade_private_4.sql @@ -1,14 +1,5 @@ BEGIN TRANSACTION; -CREATE TABLE IF NOT EXISTS "ratings" ( - "videoID" TEXT NOT NULL, - "service" TEXT NOT NULL default 'YouTube', - "type" INTEGER NOT NULL, - "userID" TEXT NOT NULL, - "timeSubmitted" INTEGER NOT NULL, - "hashedIP" TEXT NOT NULL -); - UPDATE "config" SET value = 4 WHERE key = 'version'; COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_28.sql b/databases/_upgrade_sponsorTimes_28.sql index 031436a7..c33f8890 100644 --- a/databases/_upgrade_sponsorTimes_28.sql +++ b/databases/_upgrade_sponsorTimes_28.sql @@ -1,13 +1,5 @@ BEGIN TRANSACTION; -CREATE TABLE IF NOT EXISTS "ratings" ( - "videoID" TEXT NOT NULL, - "service" TEXT NOT NULL default 'YouTube', - "type" INTEGER NOT NULL, - "count" INTEGER NOT NULL, - "hashedVideoID" TEXT NOT NULL -); - UPDATE "config" SET value = 28 WHERE key = 'version'; COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_32.sql b/databases/_upgrade_sponsorTimes_32.sql index ebba3e6e..19131930 100644 --- a/databases/_upgrade_sponsorTimes_32.sql +++ b/databases/_upgrade_sponsorTimes_32.sql @@ -12,7 +12,6 @@ ALTER TABLE "shadowBannedUsers" ADD PRIMARY KEY ("userID"); --!sqlite-ignore ALTER TABLE "unlistedVideos" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore ALTER TABLE "config" ADD PRIMARY KEY ("key"); --!sqlite-ignore ALTER TABLE "archivedSponsorTimes" ADD PRIMARY KEY ("UUID"); --!sqlite-ignore -ALTER TABLE "ratings" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore UPDATE "config" SET value = 32 WHERE key = 'version'; diff --git a/databases/_upgrade_sponsorTimes_35.sql b/databases/_upgrade_sponsorTimes_35.sql index eb10ee53..de4690a0 100644 --- a/databases/_upgrade_sponsorTimes_35.sql +++ b/databases/_upgrade_sponsorTimes_35.sql @@ -1,7 +1,5 @@ BEGIN TRANSACTION; -ALTER TABLE "titleVotes" ADD "verification" INTEGER default 0; - UPDATE "config" SET value = 35 WHERE key = 'version'; COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_37.sql b/databases/_upgrade_sponsorTimes_37.sql index 4c347542..aafbefd4 100644 --- a/databases/_upgrade_sponsorTimes_37.sql +++ b/databases/_upgrade_sponsorTimes_37.sql @@ -1,7 +1,5 @@ BEGIN TRANSACTION; -ALTER TABLE "titles" ADD UNIQUE ("videoID", "title"); --!sqlite-ignore - UPDATE "config" SET value = 37 WHERE key = 'version'; COMMIT; \ No newline at end of file diff --git a/databases/_upgrade_sponsorTimes_38.sql b/databases/_upgrade_sponsorTimes_38.sql index ac5aa961..98636a49 100644 --- a/databases/_upgrade_sponsorTimes_38.sql +++ b/databases/_upgrade_sponsorTimes_38.sql @@ -1,11 +1,5 @@ BEGIN TRANSACTION; -UPDATE "titleVotes" SET "shadowHidden" = 1 -WHERE "UUID" IN (SELECT "UUID" FROM "titles" INNER JOIN "shadowBannedUsers" "bans" ON "titles"."userID" = "bans"."userID"); - -UPDATE "thumbnailVotes" SET "shadowHidden" = 1 -WHERE "UUID" IN (SELECT "UUID" FROM "thumbnails" INNER JOIN "shadowBannedUsers" "bans" ON "thumbnails"."userID" = "bans"."userID"); - UPDATE "config" SET value = 38 WHERE key = 'version'; COMMIT; diff --git a/databases/_upgrade_sponsorTimes_39.sql b/databases/_upgrade_sponsorTimes_39.sql index 2e048c23..515abab6 100644 --- a/databases/_upgrade_sponsorTimes_39.sql +++ b/databases/_upgrade_sponsorTimes_39.sql @@ -1,11 +1,5 @@ BEGIN TRANSACTION; -ALTER TABLE "titleVotes" ADD "downvotes" INTEGER default 0; -ALTER TABLE "titleVotes" ADD "removed" INTEGER default 0; - -ALTER TABLE "thumbnailVotes" ADD "downvotes" INTEGER default 0; -ALTER TABLE "thumbnailVotes" ADD "removed" INTEGER default 0; - UPDATE "config" SET value = 39 WHERE key = 'version'; COMMIT; diff --git a/databases/_upgrade_sponsorTimes_43.sql b/databases/_upgrade_sponsorTimes_43.sql new file mode 100644 index 00000000..be2feb11 --- /dev/null +++ b/databases/_upgrade_sponsorTimes_43.sql @@ -0,0 +1,30 @@ +BEGIN TRANSACTION; + +ALTER TABLE "sponsorTimes" ADD "cid" TEXT NOT NULL DEFAULT ''; +ALTER TABLE "portVideo" ADD "cid" TEXT NOT NULL DEFAULT ''; +ALTER TABLE "archivedSponsorTimes" ADD "cid" TEXT NOT NULL DEFAULT ''; +ALTER TABLE "lockCategories" ADD "cid" TEXT NOT NULL DEFAULT ''; + +ALTER TABLE "archivedSponsorTimes" ADD "ytbID" TEXT; +ALTER TABLE "archivedSponsorTimes" ADD "ytbSegmentUUID" TEXT; +ALTER TABLE "archivedSponsorTimes" ADD "portUUID" TEXT; + +CREATE TABLE "sqlb_temp_table_43" ( + "videoID" TEXT NOT NULL, + "cid" TEXT NOT NULL DEFAULT '', + "channelID" TEXT NOT NULL, + "title" TEXT NOT NULL, + "part" INTEGER NOT NULL DEFAULT 0, + "partTitile" TEXT, + "published" NUMERIC NOT NULL, + PRIMARY KEY("videoID", "cid") +); + +INSERT INTO sqlb_temp_table_43 SELECT "videoID", '', "channelID", "title", 1, '', "published" FROM "videoInfo"; + +DROP TABLE "videoInfo"; +ALTER TABLE sqlb_temp_table_43 RENAME TO "videoInfo"; + +UPDATE "config" SET value = 43 WHERE key = 'version'; + +COMMIT; diff --git a/src/cronjob/index.ts b/src/cronjob/index.ts index d6770ea4..8b7dc50f 100644 --- a/src/cronjob/index.ts +++ b/src/cronjob/index.ts @@ -1,6 +1,7 @@ import { config } from "../config"; import { Logger } from "../utils/logger"; import { dumpDatebaseJob } from "./dumpDatabase"; +import { refreshCidJob } from "./refreshCid"; import refreshTopUserViewJob from "./refreshTopUserView"; export function startAllCrons(): void { @@ -9,6 +10,7 @@ export function startAllCrons(): void { refreshTopUserViewJob.start(); dumpDatebaseJob.start(); + refreshCidJob.start(); } else { Logger.info("Crons dissabled"); } diff --git a/src/cronjob/refreshCid.ts b/src/cronjob/refreshCid.ts new file mode 100644 index 00000000..1d693b74 --- /dev/null +++ b/src/cronjob/refreshCid.ts @@ -0,0 +1,78 @@ +import { CronJob } from "cron"; +import { saveVideoInfo } from "../dao/videoInfo"; +import { db } from "../databases/databases"; +import { DBSegment, HiddenType, SegmentUUID } from "../types/segments.model"; +import { durationEquals } from "../utils/durationUtil"; +import { getVideoDetails, VideoDetail } from "../utils/getVideoDetails"; +import { Logger } from "../utils/logger"; +import { sleep } from "../utils/timeUtil"; + +export const refreshCidJob = new CronJob("*/1 * * * *", () => refreshCid()); + +let isRunning = false; + +async function refreshCid() { + if (isRunning) { + Logger.info("refreshCid already running, skipping"); + return; + } + + isRunning = true; + const allSegments: DBSegment[] = await db.prepare("all", `SELECT * FROM "sponsorTimes" WHERE "cid" = NULL or "cid" = ''`, []); + const videoSegmentMap = new Map(); + for (const segment of allSegments) { + if (!videoSegmentMap.has(segment.videoID)) { + videoSegmentMap.set(segment.videoID, []); + } + videoSegmentMap.get(segment.videoID)?.push(segment); + } + Logger.info(`Found ${videoSegmentMap.size} videos with missing cids`); + + for (const [videoID, segments] of videoSegmentMap) { + let biliVideoDetail: VideoDetail; + try { + biliVideoDetail = await getVideoDetails(videoID); + if (biliVideoDetail === null || biliVideoDetail === undefined) { + Logger.error(`Failed to get video detail for ${videoID}`); + continue; + } + } catch (e) { + Logger.error(`Failed to get video detail for ${videoID}`); + continue; + } + + await saveVideoInfo(biliVideoDetail); + + const invalidIDs: SegmentUUID[] = []; + + if (biliVideoDetail.page.length === 1 || !!biliVideoDetail.page[0].cid) { + await db.prepare("run", `UPDATE "sponsorTimes" SET "cid" = ? WHERE "videoID" = ? AND "cid" = ''`, [ + biliVideoDetail.page[0].cid, + videoID, + ]); + invalidIDs.push( + ...segments.filter((s) => !durationEquals(s.videoDuration, biliVideoDetail.page[0].duration)).map((s) => s.UUID) + ); + } else { + // find a matching cid + for (const segment of segments) { + const possibleCids = biliVideoDetail.page.filter((p) => durationEquals(p.duration, segment.videoDuration)); + if (possibleCids.length === 1) { + await db.prepare("run", `UPDATE "sponsorTimes" SET "cid" = ? WHERE "UUID" = ?`, [possibleCids[0].cid, segment.UUID]); + } + } + } + + // Hide segments with invalid cids + if (invalidIDs.length > 0) { + await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = ? WHERE "UUID" IN (${invalidIDs.map(() => "?").join(",")})`, [ + HiddenType.Hidden, + ...invalidIDs, + ]); + } + + await sleep(10000); + } + + isRunning = false; +} diff --git a/src/dao/skipSegment.ts b/src/dao/skipSegment.ts index 3ad3c1f3..4ce0f1a2 100644 --- a/src/dao/skipSegment.ts +++ b/src/dao/skipSegment.ts @@ -1,36 +1,23 @@ import { db, privateDB } from "../databases/databases"; import { PORT_SEGMENT_USER_ID } from "../routes/postPortVideo"; import { portVideoUUID } from "../types/portVideo.model"; -import { - DBSegment, - HashedIP, - HiddenType, - Segment, - SegmentUUID, - Service, - VideoID, - VideoIDHash, - Visibility, -} from "../types/segments.model"; +import { DBSegment, HashedIP, HiddenType, Segment, SegmentUUID, Service, VideoID, VideoIDHash, Visibility } from "../types/segments.model"; import { getHash } from "../utils/getHash"; import { getPortSegmentUUID } from "../utils/getSubmissionUUID"; import { QueryCacher } from "../utils/queryCacher"; -import { skipSegmentsHashKey, skipSegmentsKey } from "../utils/redisKeys"; +import { cidListKey, skipSegmentsHashKey, skipSegmentsKey } from "../utils/redisKeys"; -export async function getSegmentsFromDBByHash( - hashedVideoIDPrefix: VideoIDHash, - service: Service -): Promise { +export async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise { const fetchFromDB = () => db.prepare( "all", - `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "hidden", "reputation", "shadowHidden", "hashedVideoID", "timeSubmitted", "description", "ytbID", "ytbSegmentUUID", "portUUID" FROM "sponsorTimes" + `SELECT "videoID", "cid", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "hidden", "reputation", "shadowHidden", "hashedVideoID", "timeSubmitted", "description", "ytbID", "ytbSegmentUUID", "portUUID" FROM "sponsorTimes" WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "startTime"`, [`${hashedVideoIDPrefix}%`, service], { useReplica: true } ) as Promise; - if (hashedVideoIDPrefix.length === 4) { + if (hashedVideoIDPrefix.length >= 4) { return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service)); } @@ -41,7 +28,7 @@ export async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Serv const fetchFromDB = () => db.prepare( "all", - `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "hidden", "reputation", "shadowHidden", "timeSubmitted", "description", "ytbID", "ytbSegmentUUID", "portUUID" FROM "sponsorTimes" + `SELECT "cid", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "hidden", "reputation", "shadowHidden", "timeSubmitted", "description", "ytbID", "ytbSegmentUUID", "portUUID" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? ORDER BY "startTime"`, [videoID, service], { useReplica: true } @@ -54,19 +41,14 @@ export async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Serv * hide segments by UUID from the same video, * provide the video id to clear redis cache */ -export async function hideSegmentsByUUID( - UUIDs: string[], - bvID: VideoID, - hiddenType = HiddenType.MismatchHidden -): Promise { +export async function hideSegmentsByUUID(UUIDs: string[], bvID: VideoID, hiddenType = HiddenType.MismatchHidden): Promise { if (UUIDs.length === 0) { return; } - await db.prepare( - "run", - `UPDATE "sponsorTimes" SET "hidden" = ? WHERE "UUID" IN (${Array(UUIDs.length).fill("?").join(",")})`, - [hiddenType, ...UUIDs] - ); + await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = ? WHERE "UUID" IN (${Array(UUIDs.length).fill("?").join(",")})`, [ + hiddenType, + ...UUIDs, + ]); QueryCacher.clearSegmentCacheByID(bvID); } @@ -192,3 +174,10 @@ export async function updateVotes(segments: DBSegment[]): Promise { const videoIDSet = new Set(segments.map((s) => s.videoID)); videoIDSet.forEach((videoID) => QueryCacher.clearSegmentCacheByID(videoID)); } + +export async function getAllCid(videoID: VideoID): Promise { + const fetchData = async (videoID: VideoID) => { + return await db.prepare("all", `SELECT DISTINCT cid FROM "sponsorTimes" WHERE "videoID" =?`, [videoID]); + }; + return await QueryCacher.get(() => fetchData(videoID), cidListKey(videoID)); +} diff --git a/src/dao/videoInfo.ts b/src/dao/videoInfo.ts index 4947d73e..46e9d7fe 100644 --- a/src/dao/videoInfo.ts +++ b/src/dao/videoInfo.ts @@ -1,17 +1,23 @@ import { db } from "../databases/databases"; -import { videoDetails } from "../utils/getVideoDetails"; +import { VideoDetail } from "../utils/getVideoDetails"; -export async function saveVideoInfo(biliVideoDetail: videoDetails) { - await db.prepare( - "run", - `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published") SELECT ?, ?, ?, ? - WHERE NOT EXISTS (SELECT 1 FROM "videoInfo" WHERE "videoID" = ?)`, - [ - biliVideoDetail.videoId, - biliVideoDetail?.authorId || "", - biliVideoDetail?.title || "", - biliVideoDetail?.published || 0, - biliVideoDetail.videoId, - ] - ); +export async function saveVideoInfo(biliVideoDetail: VideoDetail) { + for (const page of biliVideoDetail.page) { + await db.prepare( + "run", + `INSERT INTO "videoInfo" ("videoID", "cid", "channelID", "title", "part", "partTitile", "published") SELECT ?, ?, ?, ?, ?, ?, ? + WHERE NOT EXISTS (SELECT 1 FROM "videoInfo" WHERE "videoID" = ? AND "cid" = ?)`, + [ + biliVideoDetail.videoId, + page.cid, + biliVideoDetail?.authorId || "", + biliVideoDetail?.title || "", + page?.page || 0, + page?.part || "", + biliVideoDetail?.published || 0, + biliVideoDetail.videoId, + page.cid, + ] + ); + } } diff --git a/src/routes/getPortVideo.ts b/src/routes/getPortVideo.ts index c020d7d5..6c11b47f 100644 --- a/src/routes/getPortVideo.ts +++ b/src/routes/getPortVideo.ts @@ -5,18 +5,12 @@ import { getPortVideoDBByBvIDCached, hidePortVideoByUUID, } from "../dao/portVideo"; -import { - createSegmentsFromYTB, - getSegmentsFromDBByVideoID, - hideSegmentsByUUID, - saveNewSegments, - updateVotes, -} from "../dao/skipSegment"; +import { createSegmentsFromYTB, getSegmentsFromDBByVideoID, hideSegmentsByUUID, saveNewSegments, updateVotes } from "../dao/skipSegment"; import { HashedValue } from "../types/hash.model"; import { PortVideo, PortVideoDB, PortVideoInterface } from "../types/portVideo.model"; import { DBSegment, Service, VideoDuration, VideoID } from "../types/segments.model"; import { average } from "../utils/array"; -import { validate } from "../utils/bilibiliID"; +import { validate } from "../validate/bilibiliID"; import { durationEquals, durationsAllEqual } from "../utils/durationUtil"; import { getVideoDetails } from "../utils/getVideoDetails"; import { getYoutubeSegments, getYoutubeVideoDuraion } from "../utils/getYoutubeVideoSegments"; @@ -43,17 +37,14 @@ export async function updatePortedSegments(req: Request, res: Response) { export async function updateSegmentsFromSB(portVideo: PortVideoDB) { const bvID = portVideo.bvID; + const cid = portVideo.cid; const ytbID = portVideo.ytbID; const [ytbSegments, biliVideoDetail] = await Promise.all([getYoutubeSegments(ytbID), getVideoDetails(bvID, true)]); // get ytb video duration let apiYtbDuration = 0 as VideoDuration; if (ytbSegments && ytbSegments.length > 0) { - apiYtbDuration = average( - ytbSegments.filter((s) => s.videoDuration > 0).map((s) => s.videoDuration) - ) as VideoDuration; - Logger.info( - `Retrieved ${ytbSegments.length} segments from SB server. Average video duration: ${apiYtbDuration}s` - ); + apiYtbDuration = average(ytbSegments.filter((s) => s.videoDuration > 0).map((s) => s.videoDuration)) as VideoDuration; + Logger.info(`Retrieved ${ytbSegments.length} segments from SB server. Average video duration: ${apiYtbDuration}s`); } if (!apiYtbDuration) { apiYtbDuration = await getYoutubeVideoDuraion(ytbID); @@ -66,7 +57,7 @@ export async function updateSegmentsFromSB(portVideo: PortVideoDB) { // if no youtube duration is provided, dont't do anything return; } - const apiBiliDuration = biliVideoDetail?.duration as VideoDuration; + const apiBiliDuration = biliVideoDetail?.page.filter((p) => p.cid == cid)[0]?.duration as VideoDuration; if (!apiBiliDuration) { // if no bili duration is found, dont't do anything return; diff --git a/src/routes/getSearchSegments.ts b/src/routes/getSearchSegments.ts index 13780eb5..024fb22a 100644 --- a/src/routes/getSearchSegments.ts +++ b/src/routes/getSearchSegments.ts @@ -16,7 +16,7 @@ type searchSegmentResponse = { function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise { return db.prepare( "all", - `SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID", "description" FROM "sponsorTimes" + `SELECT "cid", "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID", "description" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? ORDER BY "timeSubmitted"`, [videoID, service] ) as Promise; diff --git a/src/routes/getSkipSegments.ts b/src/routes/getSkipSegments.ts index faaf43db..eed14cbe 100644 --- a/src/routes/getSkipSegments.ts +++ b/src/routes/getSkipSegments.ts @@ -1,8 +1,10 @@ import { Request, Response } from "express"; import { partition } from "lodash"; import { config } from "../config"; +import { getSegmentsFromDBByHash } from "../dao/skipSegment"; import { db, privateDB } from "../databases/databases"; -import { skipSegmentsHashKey, skipSegmentsKey, skipSegmentGroupsKey, shadowHiddenIPKey } from "../utils/redisKeys"; +import { Postgres } from "../databases/Postgres"; +import { getEtag } from "../middleware/etag"; import { SBRecord } from "../types/lib.model"; import { ActionType, @@ -21,22 +23,22 @@ import { Visibility, VotableObject, } from "../types/segments.model"; +import { shuffleArray } from "../utils/array"; +import { getHash } from "../utils/getHash"; import { getHashCache } from "../utils/getHashCache"; import { getIP } from "../utils/getIP"; +import { getService } from "../utils/getService"; import { Logger } from "../utils/logger"; +import { parseSkipSegments } from "../utils/parseSkipSegments"; +import { promiseOrTimeout } from "../utils/promise"; import { QueryCacher } from "../utils/queryCacher"; +import { shadowHiddenIPKey, skipSegmentGroupsKey } from "../utils/redisKeys"; import { getReputation } from "../utils/reputation"; -import { getService } from "../utils/getService"; -import { promiseOrTimeout } from "../utils/promise"; -import { parseSkipSegments } from "../utils/parseSkipSegments"; -import { getEtag } from "../middleware/etag"; -import { shuffleArray } from "../utils/array"; -import { Postgres } from "../databases/Postgres"; -import { getHash } from "../utils/getHash"; async function prepareCategorySegments( req: Request, videoID: VideoID, + cid: string, service: Service, segments: DBSegment[], cache: SegmentCache = { shadowHiddenSegmentIPs: {} }, @@ -109,8 +111,7 @@ async function prepareCategorySegments( cache.userHashedIP = await cache.userHashedIPPromise; } //if this isn't their ip, don't send it to them - const shouldShadowHide = - ipList?.some((shadowHiddenSegment) => shadowHiddenSegment.hashedIP === cache.userHashedIP) ?? false; + const shouldShadowHide = ipList?.some((shadowHiddenSegment) => shadowHiddenSegment.hashedIP === cache.userHashedIP) ?? false; if (shouldShadowHide) useCache = false; return shouldShadowHide; @@ -119,74 +120,21 @@ async function prepareCategorySegments( const filteredSegments = segments.filter((_, index) => shouldFilter[index]); - return (await chooseSegments(videoID, service, filteredSegments, useCache)).map((chosenSegment) => ({ - category: chosenSegment.category, - actionType: chosenSegment.actionType, - segment: [chosenSegment.startTime, chosenSegment.endTime], - UUID: chosenSegment.UUID, - locked: chosenSegment.locked, - votes: chosenSegment.votes, - videoDuration: chosenSegment.videoDuration, - userID: chosenSegment.userID, - description: chosenSegment.description, - })); -} - -async function getSegmentsByVideoID( - req: Request, - videoID: VideoID, - categories: Category[], - actionTypes: ActionType[], - requiredSegments: SegmentUUID[], - service: Service -): Promise { - const cache: SegmentCache = { shadowHiddenSegmentIPs: {} }; - - // For old clients - const forcePoiAsSkip = !actionTypes.includes(ActionType.Poi) && categories.includes("poi_highlight" as Category); - if (forcePoiAsSkip) { - actionTypes.push(ActionType.Poi); - } - - try { - const segments: DBSegment[] = (await getSegmentsFromDBByVideoID(videoID, service)).map((segment: DBSegment) => { - if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true; - return segment; - }, {}); - - const canUseCache = requiredSegments.length === 0; - let processedSegments: Segment[] = ( - await prepareCategorySegments(req, videoID, service, segments, cache, canUseCache) - ) - .filter( - (segment: Segment) => - categories.includes(segment?.category) && actionTypes.includes(segment?.actionType) - ) - .map((segment: Segment) => ({ - category: segment.category, - actionType: segment.actionType, - segment: segment.segment, - UUID: segment.UUID, - videoDuration: segment.videoDuration, - locked: segment.locked, - votes: segment.votes, - description: segment.description, - })); - - if (forcePoiAsSkip) { - processedSegments = processedSegments.map((segment) => ({ - ...segment, - actionType: segment.actionType === ActionType.Poi ? ActionType.Skip : segment.actionType, - })); - } - - return processedSegments; - } catch (err) /* istanbul ignore next */ { - if (err) { - Logger.error(err as string); - return null; - } - } + return (await chooseSegments(videoID, cid, service, filteredSegments, useCache)).map( + (chosenSegment) => + ({ + cid: chosenSegment.cid, + category: chosenSegment.category, + actionType: chosenSegment.actionType, + segment: [chosenSegment.startTime, chosenSegment.endTime], + UUID: chosenSegment.UUID, + locked: chosenSegment.locked, + votes: chosenSegment.votes, + videoDuration: chosenSegment.videoDuration, + userID: chosenSegment.userID, + description: chosenSegment.description, + } as Segment) + ); } async function getSegmentsByHash( @@ -195,7 +143,8 @@ async function getSegmentsByHash( categories: Category[], actionTypes: ActionType[], requiredSegments: SegmentUUID[], - service: Service + service: Service, + cid: string = null ): Promise> { const cache: SegmentCache = { shadowHiddenSegmentIPs: {} }; const segments: SBRecord = {}; @@ -209,42 +158,34 @@ async function getSegmentsByHash( try { type SegmentPerVideoID = SBRecord; - const segmentPerVideoID: SegmentPerVideoID = ( - await getSegmentsFromDBByHash(hashedVideoIDPrefix, service) - ).reduce((acc: SegmentPerVideoID, segment: DBSegment) => { - acc[segment.videoID] = acc[segment.videoID] || { - segments: [], - }; - if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true; + const segmentPerVideoID: SegmentPerVideoID = (await getSegmentsFromDBByHash(hashedVideoIDPrefix, service)) + .filter((segment) => !cid || segment.cid == cid) + .reduce((acc: SegmentPerVideoID, segment: DBSegment) => { + acc[`${segment.videoID},${segment.cid}`] = acc[`${segment.videoID},${segment.cid}`] || { + segments: [], + }; + if (filterRequiredSegments(segment.UUID, requiredSegments)) segment.required = true; - acc[segment.videoID].segments ??= []; - acc[segment.videoID].segments.push(segment); + acc[`${segment.videoID},${segment.cid}`].segments ??= []; + acc[`${segment.videoID},${segment.cid}`].segments.push(segment); - return acc; - }, {}); + return acc; + }, {}); await Promise.all( - Object.entries(segmentPerVideoID).map(async ([videoID, videoData]) => { + Object.entries(segmentPerVideoID).map(async ([videoIdCid, videoData]) => { const data: VideoData = { segments: [], }; const canUseCache = requiredSegments.length === 0; + const [videoID, cid] = videoIdCid.split(","); data.segments = ( - await prepareCategorySegments( - req, - videoID as VideoID, - service, - videoData.segments, - cache, - canUseCache - ) + await prepareCategorySegments(req, videoID as VideoID, cid, service, videoData.segments, cache, canUseCache) ) - .filter( - (segment: Segment) => - categories.includes(segment?.category) && actionTypes.includes(segment?.actionType) - ) + .filter((segment: Segment) => categories.includes(segment?.category) && actionTypes.includes(segment?.actionType)) .map((segment) => ({ + cid: segment.cid, category: segment.category, actionType: segment.actionType, segment: segment.segment, @@ -263,7 +204,11 @@ async function getSegmentsByHash( } if (data.segments.length > 0) { - segments[videoID] = data; + if (!segments[videoID]?.segments) { + segments[videoID] = data; + } else { + segments[videoID].segments.push(...data.segments); + } } }) ); @@ -275,36 +220,6 @@ async function getSegmentsByHash( } } -async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise { - const fetchFromDB = () => - db.prepare( - "all", - `SELECT "videoID", "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "hidden", "reputation", "shadowHidden", "hashedVideoID", "timeSubmitted", "description" FROM "sponsorTimes" - WHERE "hashedVideoID" LIKE ? AND "service" = ? ORDER BY "startTime"`, - [`${hashedVideoIDPrefix}%`, service], - { useReplica: true } - ) as Promise; - - if (hashedVideoIDPrefix.length === 4) { - return await QueryCacher.get(fetchFromDB, skipSegmentsHashKey(hashedVideoIDPrefix, service)); - } - - return await fetchFromDB(); -} - -async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise { - const fetchFromDB = () => - db.prepare( - "all", - `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "videoDuration", "hidden", "reputation", "shadowHidden", "timeSubmitted", "description" FROM "sponsorTimes" - WHERE "videoID" = ? AND "service" = ? ORDER BY "startTime"`, - [videoID, service], - { useReplica: true } - ) as Promise; - - return await QueryCacher.get(fetchFromDB, skipSegmentsKey(videoID, service)); -} - // Gets the best choice from the choices array based on their `votes` property. // amountOfChoices specifies the maximum amount of choices to return, 1 or more. // Choices are unique @@ -362,6 +277,7 @@ function getBestChoice( async function chooseSegments( videoID: VideoID, + cid: string, service: Service, segments: DBSegment[], useCache: boolean @@ -370,7 +286,7 @@ async function chooseSegments( const groups = useCache && config.useCacheForSegmentGroups - ? await QueryCacher.get(fetchData, skipSegmentGroupsKey(videoID, service)) + ? await QueryCacher.get(fetchData, skipSegmentGroupsKey(videoID, cid, service)) : await fetchData(); // Filter for only 1 item for POI categories and Full video @@ -412,8 +328,7 @@ async function buildSegmentGroups(segments: DBSegment[]): Promise 0) { currentGroup.reputation += segment.reputation; } @@ -454,11 +369,9 @@ function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegm // Since POI and Full video segments will always have <= 0 overlap, they will always be in their own groups return group.segments.some((compareSegment) => { const overlap = - Math.min(segment.endTime, compareSegment.endTime) - - Math.max(segment.startTime, compareSegment.startTime); + Math.min(segment.endTime, compareSegment.endTime) - Math.max(segment.startTime, compareSegment.startTime); const overallDuration = - Math.max(segment.endTime, compareSegment.endTime) - - Math.min(segment.startTime, compareSegment.startTime); + Math.max(segment.endTime, compareSegment.endTime) - Math.min(segment.startTime, compareSegment.startTime); const overlapPercent = overlap / overallDuration; return ( (overlapPercent >= 0.1 && @@ -498,6 +411,7 @@ function splitPercentOverlap(groups: OverlappingSegmentGroup[]): OverlappingSegm async function getSkipSegments(req: Request, res: Response): Promise { const videoID = req.query.videoID as VideoID; + const cid = req.query.cid as string; if (!videoID) { return res.status(400).send("videoID not specified"); } @@ -509,17 +423,21 @@ async function getSkipSegments(req: Request, res: Response): Promise { const { categories, actionTypes, requiredSegments, service } = parseResult; const hashedVideoID = getHash(videoID, 1).substring(0, 4) as VideoIDHash; - const allSegments = await getSegmentsByHash(req, hashedVideoID, categories, actionTypes, requiredSegments, service); + const allSegments = await getSegmentsByHash(req, hashedVideoID, categories, actionTypes, requiredSegments, service, cid); if (allSegments === null || allSegments === undefined) { return res.sendStatus(500); } - const segments = allSegments[videoID]?.segments; + let segments = allSegments[videoID]?.segments; if (!segments || segments.length === 0) { return res.sendStatus(404); } + if (cid) { + segments = segments.filter((s) => s.cid == cid); + } + await getEtag("skipSegments", videoID as string, service) .then((etag) => res.set("ETag", etag)) .catch(() => null); @@ -533,4 +451,4 @@ const filterRequiredSegments = (UUID: SegmentUUID, requiredSegments: SegmentUUID return false; }; -export { getSegmentsByVideoID, getSegmentsByHash, getSkipSegments }; +export { getSegmentsByHash, getSkipSegments }; diff --git a/src/routes/getVideoLabel.ts b/src/routes/getVideoLabel.ts index a5dacdfa..b1b68dc6 100644 --- a/src/routes/getVideoLabel.ts +++ b/src/routes/getVideoLabel.ts @@ -1,27 +1,22 @@ import { Request, Response } from "express"; -import { db } from "../databases/databases"; -import { videoLabelsHashKey, videoLabelsKey } from "../utils/redisKeys"; +import { getSegmentsFromDBByHash, getSegmentsFromDBByVideoID } from "../dao/skipSegment"; import { SBRecord } from "../types/lib.model"; -import { DBSegment, Segment, Service, VideoData, VideoID, VideoIDHash } from "../types/segments.model"; -import { Logger } from "../utils/logger"; -import { QueryCacher } from "../utils/queryCacher"; +import { ActionType, DBSegment, HiddenType, Service, VideoID, VideoIDHash, VideoLabel, VideoLabelData } from "../types/segments.model"; import { getService } from "../utils/getService"; +import { Logger } from "../utils/logger"; -function transformDBSegments(segments: DBSegment[]): Segment[] { +function transformDBSegments(segments: DBSegment[]): VideoLabel[] { return segments.map((chosenSegment) => ({ + cid: chosenSegment.cid, category: chosenSegment.category, - actionType: chosenSegment.actionType, - segment: [chosenSegment.startTime, chosenSegment.endTime], UUID: chosenSegment.UUID, locked: chosenSegment.locked, votes: chosenSegment.votes, videoDuration: chosenSegment.videoDuration, - userID: chosenSegment.userID, - description: chosenSegment.description })); } -async function getLabelsByVideoID(videoID: VideoID, service: Service): Promise { +async function getLabelsByVideoID(videoID: VideoID, service: Service): Promise { try { const segments: DBSegment[] = await getSegmentsFromDBByVideoID(videoID, service); return chooseSegment(segments); @@ -33,8 +28,8 @@ async function getLabelsByVideoID(videoID: VideoID, service: Service): Promise> { - const segments: SBRecord = {}; +async function getLabelsByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise> { + const segments: SBRecord = {}; try { type SegmentWithHashPerVideoID = SBRecord; @@ -53,7 +48,7 @@ async function getLabelsByHash(hashedVideoIDPrefix: VideoIDHash, service: Servic }, {}); for (const [videoID, videoData] of Object.entries(segmentPerVideoID)) { - const data: VideoData = { + const data: VideoLabelData = { segments: chooseSegment(videoData.segments), }; @@ -69,37 +64,9 @@ async function getLabelsByHash(hashedVideoIDPrefix: VideoIDHash, service: Servic } } -async function getSegmentsFromDBByHash(hashedVideoIDPrefix: VideoIDHash, service: Service): Promise { - const fetchFromDB = () => db - .prepare( - "all", - `SELECT "startTime", "endTime", "videoID", "votes", "locked", "UUID", "userID", "category", "actionType", "hashedVideoID", "description" FROM "sponsorTimes" - WHERE "hashedVideoID" LIKE ? AND "service" = ? AND "actionType" = 'full' AND "hidden" = 0 AND "shadowHidden" = 0`, - [`${hashedVideoIDPrefix}%`, service] - ) as Promise; - - if (hashedVideoIDPrefix.length === 3) { - return await QueryCacher.get(fetchFromDB, videoLabelsHashKey(hashedVideoIDPrefix, service)); - } - - return await fetchFromDB(); -} - -async function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise { - const fetchFromDB = () => db - .prepare( - "all", - `SELECT "startTime", "endTime", "votes", "locked", "UUID", "userID", "category", "actionType", "description" FROM "sponsorTimes" - WHERE "videoID" = ? AND "service" = ? AND "actionType" = 'full' AND "hidden" = 0 AND "shadowHidden" = 0`, - [videoID, service] - ) as Promise; - - return await QueryCacher.get(fetchFromDB, videoLabelsKey(videoID, service)); -} - -function chooseSegment(choices: T[]): Segment[] { +function chooseSegment(choices: T[]): VideoLabel[] { // filter out -2 segments - choices = choices.filter((segment) => segment.votes > -2); + choices = choices.filter(segment => segment.actionType == ActionType.Full && segment.votes > -2 && segment.hidden == HiddenType.Show); const results = []; // trivial decisions if (choices.length === 0) { @@ -125,7 +92,7 @@ function chooseSegment(choices: T[]): Segment[] { return transformDBSegments(results); } -async function handleGetLabel(req: Request, res: Response): Promise { +async function handleGetLabel(req: Request, res: Response): Promise { const videoID = req.query.videoID as VideoID; if (!videoID) { res.status(400).send("videoID not specified"); @@ -160,7 +127,5 @@ async function endpoint(req: Request, res: Response): Promise { } export { - getLabelsByVideoID, - getLabelsByHash, - endpoint + endpoint, getLabelsByHash, getLabelsByVideoID }; diff --git a/src/routes/postPortVideo.ts b/src/routes/postPortVideo.ts index 81ea7e3d..b66e8270 100644 --- a/src/routes/postPortVideo.ts +++ b/src/routes/postPortVideo.ts @@ -4,8 +4,8 @@ import { db, privateDB } from "../databases/databases"; import { HashedUserID } from "../types/user.model"; import { getHashCache, getHashedIP } from "../utils/getHashCache"; import { config } from "../config"; -import * as youtubeID from "../utils/youtubeID"; -import * as biliID from "../utils/bilibiliID"; +import * as youtubeID from "../validate/youtubeID"; +import * as biliID from "../validate/bilibiliID"; import { getVideoDetails } from "../utils/getVideoDetails"; import { parseUserAgentFromHeaders } from "../utils/userAgent"; import { getMatchVideoUUID, getPortSegmentUUID } from "../utils/getSubmissionUUID"; @@ -39,10 +39,10 @@ export const PORT_SEGMENT_USER_ID = "PORT"; export async function postPortVideo(req: Request, res: Response): Promise { const bvID = req.query.bvID || req.body.bvID; + let cid = req.query.cid || req.body.cid; const ytbID = req.query.ytbID || req.body.ytbID; const paramUserID = req.query.userID || req.body.userID; - const paramBiliDuration: VideoDuration = (parseFloat(req.query.biliDuration || req.body.biliDuration) || - 0) as VideoDuration; + const paramBiliDuration: VideoDuration = (parseFloat(req.query.biliDuration || req.body.biliDuration) || 0) as VideoDuration; const rawIP = getIP(req); const hashedBvID = getHash(bvID, 1); @@ -64,12 +64,19 @@ export async function postPortVideo(req: Request, res: Response): Promise p.cid == cid); + if (pageDetail.length == 0) { + return res.status(400).send("分p视频无法获取cid，请使用0.5.0以上版本的插件！"); + } + cid = pageDetail[0].cid; + } + // get ytb video duration let ytbDuration = 0 as VideoDuration; if (ytbSegments && ytbSegments.length > 0) { - ytbDuration = average( - ytbSegments.filter((s) => s.videoDuration > 0).map((s) => s.videoDuration) - ) as VideoDuration; + ytbDuration = average(ytbSegments.filter((s) => s.videoDuration > 0).map((s) => s.videoDuration)) as VideoDuration; Logger.info(`Retrieved ${ytbSegments.length} segments from SB server. Average video duration: ${ytbDuration}s`); } if (!ytbDuration) { @@ -83,7 +90,12 @@ export async function postPortVideo(req: Request, res: Response): Promise p.cid == cid).length == 0) { + return res.status(400).send("cid有误！请刷新页面再试"); + } + // check duration + const apiBiliDuration = biliVideoDetail?.page.filter((p) => p.cid == cid)[0].duration as VideoDuration; if (!paramBiliDuration || !apiBiliDuration) { lock.unlock(); return res.status(400).send(`无法获取B站视频信息，请重试。 @@ -111,9 +123,7 @@ export async function postPortVideo(req: Request, res: Response): Promise - port.ytbID == ytbID && - durationsAllEqual([port.biliDuration, port.ytbDuration, apiBiliDuration, ytbDuration]) + (port) => port.ytbID == ytbID && durationsAllEqual([port.biliDuration, port.ytbDuration, apiBiliDuration, ytbDuration]) ); if (exactMatches.length > 0) { lock.unlock(); @@ -204,11 +214,12 @@ export async function postPortVideo(req: Request, res: Response): Promise Logger.error(`sending webhooks: ${e}`)); - - // If it is a first time submission - // Then send a notification to discord - if (config.discordFirstTimeSubmissionsWebhookURL === null || userSubmissionCountRow.submissionCount > 1) return; - - axios.post(config.discordFirstTimeSubmissionsWebhookURL, { - embeds: [{ - title: apiVideoDetails.title, - url: `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`, - description: `Submission ID: ${UUID}\ - \n\nTimestamp: \ - ${getFormattedTime(startTime)} to ${getFormattedTime(endTime)}\ - \n\nCategory: ${segmentInfo.category}`, - color: 10813440, - author: { - name: userID, - }, - thumbnail: { - url: getMaxResThumbnail(videoID), - }, - }], - }) - .then(res => { - if (res.status >= 400) { - Logger.error("Error sending first time submission Discord hook"); - Logger.error(JSON.stringify(res)); - Logger.error("\n"); - } - }) - .catch(err => { - Logger.error("Failed to send first time submission Discord hook."); - Logger.error(JSON.stringify(err)); - Logger.error("\n"); - }); - } -} - // callback: function(reject: "String containing reason the submission was rejected") // returns: string when an error, false otherwise // Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return // false for a pass - it was confusing and lead to this bug - any use of this function in // the future could have the same problem. -async function autoModerateSubmission(apiVideoDetails: videoDetails, - submission: { videoID: VideoID; userID: HashedUserID; segments: IncomingSegment[], service: Service, videoDuration: number }) { +async function autoModerateSubmission( + apiVideoDetails: VideoDetail, + submission: { + videoID: VideoID; + cid: string; + userID: HashedUserID; + segments: IncomingSegment[]; + service: Service; + videoDuration: number; + } +) { + // check if cid exist + const pageDetail = apiVideoDetails.page.filter((p) => p.cid === submission.cid); + if (!submission.cid) { + return "分p视频无法获取cid，请使用0.5.0以上版本的插件！"; + } + if (pageDetail.length == 0) { + return "分P视频cid错误"; + } // get duration from API - const apiDuration = apiVideoDetails.duration; + const apiDuration = pageDetail[0].duration; + // if API fail or returns 0, get duration from client const duration = apiDuration || submission.videoDuration; // return false on undefined or 0 @@ -131,16 +83,20 @@ async function autoModerateSubmission(apiVideoDetails: videoDetails, const segments = submission.segments; // map all times to float array - const allSegmentTimes = segments.filter((s) => s.actionType !== ActionType.Chapter) - .map(segment => [parseFloat(segment.segment[0]), parseFloat(segment.segment[1])]); + const allSegmentTimes = segments + .filter((s) => s.actionType !== ActionType.Chapter) + .map((segment) => [parseFloat(segment.segment[0]), parseFloat(segment.segment[1])]); // add previous submissions by this user - const allSubmittedByUser = await db.prepare("all", `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? AND "videoID" = ? AND "votes" > -1 AND "actionType" != 'chapter' AND "hidden" = 0` - , [submission.userID, submission.videoID]) as { startTime: string, endTime: string }[]; + const allSubmittedByUser = (await db.prepare( + "all", + `SELECT "startTime", "endTime" FROM "sponsorTimes" WHERE "userID" = ? AND "videoID" = ? AND "votes" > -1 AND "actionType" != 'chapter' AND "hidden" = 0`, + [submission.userID, submission.videoID] + )) as { startTime: string; endTime: string }[]; if (allSubmittedByUser) { //add segments the user has previously submitted - const allSubmittedTimes = allSubmittedByUser.map((segment) => [parseFloat(segment.startTime), parseFloat(segment.endTime)]); + const allSubmittedTimes = allSubmittedByUser.map((segment) => [parseFloat(segment.startTime), parseFloat(segment.endTime)]); allSegmentTimes.push(...allSubmittedTimes); } @@ -160,35 +116,44 @@ async function autoModerateSubmission(apiVideoDetails: videoDetails, async function checkUserActiveWarning(userID: HashedUserID): Promise { const MILLISECONDS_IN_HOUR = 3600000; const now = Date.now(); - const warnings = (await db.prepare("all", - `SELECT "reason" + const warnings = ( + (await db.prepare( + "all", + `SELECT "reason" FROM warnings WHERE "userID" = ? AND "issueTime" > ? AND enabled = 1 AND type = 0 ORDER BY "issueTime" DESC`, - [ - userID, - Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR)) - ], - ) as {reason: string}[]).sort((a, b) => (b?.reason?.length ?? 0) - (a?.reason?.length ?? 0)); + [userID, Math.floor(now - config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR)] + )) as { reason: string }[] + ).sort((a, b) => (b?.reason?.length ?? 0) - (a?.reason?.length ?? 0)); if (warnings?.length >= config.maxNumberOfActiveWarnings) { - const defaultMessage = "Submission rejected due to a tip from a moderator. This means that we noticed you were making some common mistakes" - + " that are not malicious, and we just want to clarify the rules. " - + "Could you please send a message in discord.gg/SponsorBlock or matrix.to/#/#sponsor:ajay.app so we can further help you? " - + `Your userID is ${userID}.`; + const defaultMessage = + "Submission rejected due to a tip from a moderator. This means that we noticed you were making some common mistakes" + + " that are not malicious, and we just want to clarify the rules. " + + "Could you please send a message in discord.gg/SponsorBlock or matrix.to/#/#sponsor:ajay.app so we can further help you? " + + `Your userID is ${userID}.`; return { pass: false, errorMessage: defaultMessage + (warnings[0]?.reason?.length > 0 ? `\n\nTip message: '${warnings[0].reason}'` : ""), - errorCode: 403 + errorCode: 403, }; } return CHECK_PASS; } -async function checkInvalidFields(videoID: VideoID, userID: UserID, hashedUserID: HashedUserID - , segments: IncomingSegment[], videoDurationParam: number, userAgent: string, service: Service): Promise { +async function checkInvalidFields( + videoID: VideoID, + cid: string, + userID: UserID, + hashedUserID: HashedUserID, + segments: IncomingSegment[], + videoDurationParam: number, + userAgent: string, + service: Service +): Promise { const invalidFields = []; const errors = []; if (typeof videoID !== "string" || videoID?.length == 0) { @@ -198,14 +163,25 @@ async function checkInvalidFields(videoID: VideoID, userID: UserID, hashedUserID const sanitizedVideoID = biliID.validate(videoID) ? videoID : biliID.sanitize(videoID); if (!biliID.validate(sanitizedVideoID)) { invalidFields.push("videoID"); - errors.push("YouTube videoID could not be extracted"); + errors.push("无法提取BVID"); } } - const minLength = config.minUserIDLength; - if (typeof userID !== "string" || userID?.length < minLength) { + + let pass: boolean; + let errorMessage: string; + + ({ pass, errorMessage } = validatePrivateUserID(userID)); + if (!pass) { invalidFields.push("userID"); - if (userID?.length < minLength) errors.push(`userID must be at least ${minLength} characters long`); + errors.push(errorMessage); + } + + ({ pass: pass, errorMessage: errorMessage } = validateCid(cid)); + if (!pass) { + invalidFields.push("cid"); + errors.push(errorMessage); } + if (!Array.isArray(segments) || segments.length == 0) { invalidFields.push("segments"); } @@ -213,13 +189,14 @@ async function checkInvalidFields(videoID: VideoID, userID: UserID, hashedUserID for (const segmentPair of segments) { const startTime = segmentPair.segment[0]; const endTime = segmentPair.segment[1]; - if ((typeof startTime === "string" && startTime.includes(":")) || - (typeof endTime === "string" && endTime.includes(":"))) { + if ((typeof startTime === "string" && startTime.includes(":")) || (typeof endTime === "string" && endTime.includes(":"))) { invalidFields.push("segment time"); } - if (typeof segmentPair.description !== "string" - || (segmentPair.description.length !== 0 && segmentPair.actionType !== ActionType.Chapter)) { + if ( + typeof segmentPair.description !== "string" || + (segmentPair.description.length !== 0 && segmentPair.actionType !== ActionType.Chapter) + ) { invalidFields.push("segment description"); } @@ -229,7 +206,9 @@ async function checkInvalidFields(videoID: VideoID, userID: UserID, hashedUserID const permission = await canSubmit(hashedUserID, segmentPair.category); if (!permission.canSubmit) { - Logger.warn(`Rejecting submission due to lack of permissions for category ${segmentPair.category}: ${segmentPair.segment} ${hashedUserID} ${videoID} ${videoDurationParam} ${userAgent}`); + Logger.warn( + `Rejecting submission due to lack of permissions for category ${segmentPair.category}: ${segmentPair.segment} ${hashedUserID} ${videoID} ${videoDurationParam} ${userAgent}` + ); invalidFields.push(`permission to submit ${segmentPair.category}`); errors.push(permission.reason); } @@ -242,16 +221,24 @@ async function checkInvalidFields(videoID: VideoID, userID: UserID, hashedUserID return { pass: false, errorMessage: `No valid ${formattedFields}.${formattedErrors}`, - errorCode: 400 + errorCode: 400, }; } return CHECK_PASS; } -async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, userID: HashedUserID, videoID: VideoID, - segments: IncomingSegment[], service: Service, isVIP: boolean, isTempVIP: boolean, lockedCategoryList: Array): Promise { - +async function checkEachSegmentValid( + rawIP: IPAddress, + paramUserID: UserID, + userID: HashedUserID, + videoID: VideoID, + segments: IncomingSegment[], + service: Service, + isVIP: boolean, + isTempVIP: boolean, + lockedCategoryList: Array +): Promise { for (let i = 0; i < segments.length; i++) { if (segments[i] === undefined || segments[i].segment === undefined || segments[i].category === undefined) { //invalid request @@ -263,16 +250,20 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user } // Reject segment if it's in the locked categories list - const lockIndex = lockedCategoryList.findIndex(c => segments[i].category === c.category && segments[i].actionType === c.actionType); + const lockIndex = lockedCategoryList.findIndex( + (c) => segments[i].category === c.category && segments[i].actionType === c.actionType + ); if (!isVIP && lockIndex !== -1) { QueryCacher.clearSegmentCache({ videoID, hashedVideoID: await getHashCache(videoID, 1), service, - userID + userID, }); - Logger.warn(`Caught a submission for a locked category. userID: '${userID}', videoID: '${videoID}', category: '${segments[i].category}', times: ${segments[i].segment}`); + Logger.warn( + `Caught a submission for a locked category. userID: '${userID}', videoID: '${videoID}', category: '${segments[i].category}', times: ${segments[i].segment}` + ); return { pass: false, errorCode: 403, @@ -281,9 +272,13 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user `'${segments[i].category}'\n` + `${lockedCategoryList[lockIndex].reason?.length !== 0 ? `\nReason: '${lockedCategoryList[lockIndex].reason}'\n` : ""}` + `You may need to refresh if you don't see the segments.\n` + - `${(segments[i].category === "sponsor" ? "\nMaybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " + - "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n" : "")}` + - `\nIf you believe this is incorrect, please contact someone on chat.sponsor.ajay.app, discord.gg/SponsorBlock or matrix.to/#/#sponsor:ajay.app` + `${ + segments[i].category === "sponsor" + ? "\nMaybe the segment you are submitting is a different category that you have not enabled and is not a sponsor. " + + "Categories that aren't sponsor, such as self-promotion can be enabled in the options.\n" + : "" + }` + + `\nIf you believe this is incorrect, please contact someone on chat.sponsor.ajay.app, discord.gg/SponsorBlock or matrix.to/#/#sponsor:ajay.app`, }; } @@ -299,14 +294,23 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user const startTime = parseFloat(segments[i].segment[0]); const endTime = parseFloat(segments[i].segment[1]); - if (isNaN(startTime) || isNaN(endTime) - || startTime === Infinity || endTime === Infinity || startTime < 0 || startTime > endTime - || (segments[i].actionType !== ActionType.Poi - && segments[i].actionType !== ActionType.Full && startTime === endTime) - || (segments[i].actionType === ActionType.Poi && startTime !== endTime) - || (segments[i].actionType === ActionType.Full && (startTime !== 0 || endTime !== 0))) { + if ( + isNaN(startTime) || + isNaN(endTime) || + startTime === Infinity || + endTime === Infinity || + startTime < 0 || + startTime > endTime || + (segments[i].actionType !== ActionType.Poi && segments[i].actionType !== ActionType.Full && startTime === endTime) || + (segments[i].actionType === ActionType.Poi && startTime !== endTime) || + (segments[i].actionType === ActionType.Full && (startTime !== 0 || endTime !== 0)) + ) { //invalid request - return { pass: false, errorMessage: "One of your segments times are invalid (too short, endTime before startTime, etc.)", errorCode: 400 }; + return { + pass: false, + errorMessage: "One of your segments times are invalid (too short, endTime before startTime, etc.)", + errorCode: 400, + }; } // Check for POI segments before some seconds @@ -314,15 +318,23 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user return { pass: false, errorMessage: `POI cannot be that early`, errorCode: 400 }; } - if (!(isVIP || isTempVIP) && segments[i].category === "sponsor" - && segments[i].actionType === ActionType.Skip && (endTime - startTime) < 1) { + if ( + !(isVIP || isTempVIP) && + segments[i].category === "sponsor" && + segments[i].actionType === ActionType.Skip && + endTime - startTime < 1 + ) { // Too short return { pass: false, errorMessage: "Segments must be longer than 1 second long", errorCode: 400 }; } //check if this info has already been submitted before - const duplicateCheck2Row = await db.prepare("get", `SELECT "UUID" FROM "sponsorTimes" WHERE "startTime" = ? - and "endTime" = ? and "category" = ? and "actionType" = ? and "description" = ? and "videoID" = ? and "service" = ?`, [startTime, endTime, segments[i].category, segments[i].actionType, segments[i].description, videoID, service]); + const duplicateCheck2Row = await db.prepare( + "get", + `SELECT "UUID" FROM "sponsorTimes" WHERE "startTime" = ? + and "endTime" = ? and "category" = ? and "actionType" = ? and "description" = ? and "videoID" = ? and "service" = ?`, + [startTime, endTime, segments[i].category, segments[i].actionType, segments[i].description, videoID, service] + ); if (duplicateCheck2Row) { segments[i].ignoreSegment = true; @@ -335,122 +347,122 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user } if (segments.every((s) => s.ignoreSegment && s.actionType !== ActionType.Full)) { - return { pass: false, errorMessage: "Segment has already been submitted before.", errorCode: 409 }; + return { pass: false, errorMessage: "该片段已经被提交！", errorCode: 409 }; } return CHECK_PASS; } -async function checkByAutoModerator(videoID: VideoID, userID: HashedUserID, segments: IncomingSegment[], service: Service, apiVideoDetails: videoDetails, videoDuration: number): Promise { +async function checkByAutoModerator( + videoID: VideoID, + cid: string, + userID: HashedUserID, + segments: IncomingSegment[], + service: Service, + apiVideoDetails: VideoDetail, + videoDuration: number +): Promise { // Auto moderator check if (service == Service.YouTube) { - const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { videoID, userID, segments, service, videoDuration }); + const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { + videoID, + cid, + userID, + segments, + service, + videoDuration, + }); if (autoModerateResult) { return { pass: false, errorCode: 403, - errorMessage: `Submissions rejected: ${autoModerateResult} If this is an issue, send a message on Discord.` + errorMessage: `Submissions rejected: ${autoModerateResult} If this is an issue, send a message on Discord.`, }; } } return CHECK_PASS; } -async function updateDataIfVideoDurationChange(videoID: VideoID, service: Service, videoDuration: VideoDuration, videoDurationParam: VideoDuration) { - let lockedCategoryList = await db.prepare("all", 'SELECT category, "actionType", reason from "lockCategories" where "videoID" = ? AND "service" = ?', [videoID, service]); +async function updateDataIfVideoDurationChange( + videoID: VideoID, + cid: string, + service: Service, + videoDuration: VideoDuration, + videoDurationParam: VideoDuration +) { + let lockedCategoryList = await db.prepare( + "all", + 'SELECT category, "actionType", reason from "lockCategories" where "videoID" = ? AND "service" = ?', + [videoID, service] + ); - const previousSubmissions = await db.prepare("all", - `SELECT "videoDuration", "UUID" + let previousSubmissions = (await db.prepare( + "all", + `SELECT "videoDuration", "UUID", "cid" FROM "sponsorTimes" WHERE "videoID" = ? AND "service" = ? AND "hidden" = 0 AND "shadowHidden" = 0 AND "actionType" != 'full' AND "votes" > -2 AND "videoDuration" != 0`, [videoID, service] - ) as {videoDuration: VideoDuration, UUID: SegmentUUID}[]; + )) as { videoDuration: VideoDuration; UUID: SegmentUUID; cid: string }[]; // If the video's duration is changed, then the video should be unlocked and old submissions should be hidden - const videoDurationChanged = (videoDuration: number) => videoDuration != 0 - && previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); + const videoDurationChanged = (videoDuration: number) => + videoDuration != 0 && + previousSubmissions.length > 0 && + !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2); + + // Don't use cache if we don't know the video duration, or the client claims that it has changed + const ignoreCache = !cid || !videoDurationParam || previousSubmissions.length === 0 || videoDurationChanged(videoDurationParam); + const apiVideoDetails: VideoDetail = await getVideoDetails(videoID, ignoreCache); + + // if video only has 1 p, use that + if (!cid && apiVideoDetails?.page.length == 1) { + cid = apiVideoDetails.page[0].cid; + } - let apiVideoDetails: videoDetails = null; - if (service == Service.YouTube) { - // Don't use cache if we don't know the video duration, or the client claims that it has changed - const ignoreCache = !videoDurationParam || previousSubmissions.length === 0 || videoDurationChanged(videoDurationParam); - apiVideoDetails = await getVideoDetails(videoID, ignoreCache); + // check cid is valid or not + if (apiVideoDetails.page.filter((p) => p.cid == cid).length == 0) { + return false; } - const apiVideoDuration = apiVideoDetails?.duration as VideoDuration; - if (!videoDurationParam || (apiVideoDuration && Math.abs(videoDurationParam - apiVideoDuration) > 2)) { + + previousSubmissions = previousSubmissions.filter((s) => s.cid == cid); + + const apiVideoDuration = apiVideoDetails.page.filter((p) => p.cid == cid)[0].duration as VideoDuration; + if (!videoDurationParam || (apiVideoDuration && !durationEquals(videoDurationParam, apiVideoDuration))) { // If api duration is far off, take that one instead (it is only precise to seconds, not millis) - videoDuration = apiVideoDuration || 0 as VideoDuration; + videoDuration = apiVideoDuration || (0 as VideoDuration); } // Only treat as difference if both the api duration and submitted duration have changed if (videoDurationChanged(videoDuration) && (!videoDurationParam || videoDurationChanged(videoDurationParam))) { // Hide all previous submissions - await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = 1 - WHERE "videoID" = ? AND "service" = ? AND "videoDuration" != ? + await db.prepare( + "run", + `UPDATE "sponsorTimes" SET "hidden" = 1 + WHERE "videoID" = ? AND "cid" = ? AND "service" = ? AND "videoDuration" != ? AND "hidden" = 0 AND "shadowHidden" = 0 AND "actionType" != 'full' AND "votes" > -2`, - [videoID, service, videoDuration]); + [videoID, cid, service, videoDuration] + ); lockedCategoryList = []; deleteLockCategories(videoID, null, null, service).catch((e) => Logger.error(`deleting lock categories: ${e}`)); } return { + cid, videoDuration, apiVideoDetails, - lockedCategoryList + lockedCategoryList, }; } -// Disable max submissions for now -// Disable IP ratelimiting for now -// eslint-disable-next-line @typescript-eslint/no-unused-vars -async function checkRateLimit(userID:string, videoID: VideoID, service: Service, timeSubmitted: number, hashedIP: string, options: { - enableCheckByIP: boolean; - enableCheckByUserID: boolean; -} = { - enableCheckByIP: false, - enableCheckByUserID: false -}): Promise { - const yesterday = timeSubmitted - 86400000; - - if (options.enableCheckByIP) { - //check to see if this ip has submitted too many sponsors today - const rateLimitCheckRow = await privateDB.prepare("get", `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "hashedIP" = ? AND "videoID" = ? AND "timeSubmitted" > ? AND "service" = ?`, [hashedIP, videoID, yesterday, service]); - - if (rateLimitCheckRow.count >= 10) { - //too many sponsors for the same video from the same ip address - return { - pass: false, - errorCode: 429, - errorMessage: "Have submited many sponsors for the same video." - }; - } - } - - if (options.enableCheckByUserID) { - //check to see if the user has already submitted sponsors for this video - const duplicateCheckRow = await db.prepare("get", `SELECT COUNT(*) as count FROM "sponsorTimes" WHERE "userID" = ? and "videoID" = ?`, [userID, videoID]); - - if (duplicateCheckRow.count >= 16) { - //too many sponsors for the same video from the same user - return { - pass: false, - errorCode: 429, - errorMessage: "Have submited many sponsors for the same video." - }; - } - } - - return CHECK_PASS; -} - function proxySubmission(req: Request) { - axios.post(`${config.proxySubmission}/api/skipSegments?userID=${req.query.userID}&videoID=${req.query.videoID}`, req.body) - .then(res => { + axios + .post(`${config.proxySubmission}/api/skipSegments?userID=${req.query.userID}&videoID=${req.query.videoID}`, req.body) + .then((res) => { Logger.debug(`Proxy Submission: ${res.status} (${res.data})`); }) .catch(() => { @@ -460,6 +472,7 @@ function proxySubmission(req: Request) { function preprocessInput(req: Request) { const videoID = req.query.videoID || req.body.videoID; + const cid = req.query.cid || req.body.cid; const userID = req.query.userID || req.body.userID; const service = getService(req.query.service, req.body.service); const videoDurationParam: VideoDuration = (parseFloat(req.query.videoDuration || req.body.videoDuration) || 0) as VideoDuration; @@ -468,26 +481,28 @@ function preprocessInput(req: Request) { let segments = req.body.segments as IncomingSegment[]; if (segments === undefined) { // Use query instead - segments = [{ - segment: [req.query.startTime as string, req.query.endTime as string], - category: req.query.category as Category, - actionType: (req.query.actionType as ActionType) ?? ActionType.Skip, - description: req.query.description as string || "", - }]; + segments = [ + { + segment: [req.query.startTime as string, req.query.endTime as string], + category: req.query.category as Category, + actionType: (req.query.actionType as ActionType) ?? ActionType.Skip, + description: (req.query.description as string) || "", + }, + ]; } // Add default action type segments.forEach((segment) => { - if (!Object.values(ActionType).some((val) => val === segment.actionType)){ + if (!Object.values(ActionType).some((val) => val === segment.actionType)) { segment.actionType = ActionType.Skip; } segment.description ??= ""; - segment.segment = segment.segment.map((time) => typeof segment.segment[0] === "string" ? time?.replace(",", ".") : time); + segment.segment = segment.segment.map((time) => (typeof segment.segment[0] === "string" ? time?.replace(",", ".") : time)); }); const userAgent = req.query.userAgent ?? req.body.userAgent ?? parseUserAgentFromHeaders(req.headers) ?? ""; - return { videoID, userID, service, videoDuration, videoDurationParam, segments, userAgent }; + return { videoID, cid, userID, service, videoDuration, videoDurationParam, segments, userAgent }; } export async function postSkipSegments(req: Request, res: Response): Promise { @@ -496,7 +511,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise((prev, val) => `${prev} ${val.category}`, "")}', times: ${segments.reduce((prev, val) => `${prev} ${val.segment}`, "")}`); + Logger.warn( + `Caught a submission for a warned user. userID: '${userID}', videoID: '${videoID}', category: '${segments.reduce( + (prev, val) => `${prev} ${val.category}`, + "" + )}', times: ${segments.reduce((prev, val) => `${prev} ${val.segment}`, "")}` + ); return res.status(userWarningCheckResult.errorCode).send(userWarningCheckResult.errorMessage); } @@ -522,23 +551,45 @@ export async function postSkipSegments(req: Request, res: Response): Promise Logger.error(`call send webhooks ${e}`)); - } - return res.json(newSegments); } catch (err) { Logger.error(err as string); @@ -650,10 +722,8 @@ function mergeTimeSegments(ranges: number[][]) { let last: number[]; ranges.forEach(function (r) { - if (!last || r[0] > last[1]) - result.push(last = r); - else if (r[1] > last[1]) - last[1] = r[1]; + if (!last || r[0] > last[1]) result.push((last = r)); + else if (r[1] > last[1]) last[1] = r[1]; }); return result; diff --git a/src/routes/voteOnPortVideo.ts b/src/routes/voteOnPortVideo.ts index bda147e3..29c99509 100644 --- a/src/routes/voteOnPortVideo.ts +++ b/src/routes/voteOnPortVideo.ts @@ -4,7 +4,7 @@ import { db, privateDB } from "../databases/databases"; import { PortVideoDB, PortVideoVotesDB, portVideoUUID } from "../types/portVideo.model"; import { HiddenType, IPAddress, VideoID, VoteType } from "../types/segments.model"; import { HashedUserID, UserID } from "../types/user.model"; -import { validate } from "../utils/bilibiliID"; +import { validate } from "../validate/bilibiliID"; import { getHash } from "../utils/getHash"; import { getHashCache, getHashedIP } from "../utils/getHashCache"; import { getIP } from "../utils/getIP"; diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts index 22ed73f8..d1b50177 100644 --- a/src/routes/voteOnSponsorTime.ts +++ b/src/routes/voteOnSponsorTime.ts @@ -1,22 +1,34 @@ +import axios from "axios"; import { Request, Response } from "express"; -import { Logger } from "../utils/logger"; -import { isUserVIP } from "../utils/isUserVIP"; -import { isUserTempVIP } from "../utils/isUserTempVIP"; -import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi"; +import { config } from "../config"; import { db, privateDB } from "../databases/databases"; -import { dispatchEvent, getVoteAuthor, getVoteAuthorRaw } from "../utils/webhookUtils"; +import { + ActionType, + Category, + DBSegment, + HashedIP, + IPAddress, + SegmentUUID, + Service, + VideoDuration, + VideoID, + VideoIDHash, + VoteType, +} from "../types/segments.model"; +import { HashedUserID, UserID } from "../types/user.model"; +import { checkBanStatus } from "../utils/checkBan"; import { getFormattedTime } from "../utils/getFormattedTime"; -import { getIP } from "../utils/getIP"; import { getHashCache } from "../utils/getHashCache"; -import { config } from "../config"; -import { HashedUserID, UserID } from "../types/user.model"; -import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash, VideoDuration, ActionType, VoteType } from "../types/segments.model"; +import { getIP } from "../utils/getIP"; +import { getVideoDetails, VideoDetail } from "../utils/getVideoDetails"; +import { isUserTempVIP } from "../utils/isUserTempVIP"; +import { isUserVIP } from "../utils/isUserVIP"; +import { Logger } from "../utils/logger"; import { QueryCacher } from "../utils/queryCacher"; -import axios from "axios"; -import { getVideoDetails, videoDetails } from "../utils/getVideoDetails"; -import { deleteLockCategories } from "./deleteLockCategories"; import { acquireLock } from "../utils/redisLock"; -import { checkBanStatus } from "../utils/checkBan"; +import { dispatchEvent, getVoteAuthor, getVoteAuthorRaw } from "../utils/webhookUtils"; +import { getMaxResThumbnail } from "../utils/youtubeApi"; +import { deleteLockCategories } from "./deleteLockCategories"; const voteTypes = { normal: 0, @@ -25,15 +37,15 @@ const voteTypes = { enum VoteWebhookType { Normal, - Rejected + Rejected, } interface FinalResponse { - blockVote: boolean, - finalStatus: number - finalMessage: string, - webhookType: VoteWebhookType, - webhookMessage: string + blockVote: boolean; + finalStatus: number; + finalMessage: string; + webhookType: VoteWebhookType; + webhookMessage: string; } interface VoteData { @@ -55,16 +67,21 @@ interface VoteData { finalResponse: FinalResponse; } -const videoDurationChanged = (segmentDuration: number, APIDuration: number) => (APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2); +const videoDurationChanged = (segmentDuration: number, APIDuration: number) => + APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2; async function updateSegmentVideoDuration(UUID: SegmentUUID) { - const { videoDuration, videoID, service } = await db.prepare("get", `select "videoDuration", "videoID", "service" from "sponsorTimes" where "UUID" = ?`, [UUID]); - let apiVideoDetails: videoDetails = null; + const { videoDuration, videoID, cid, service } = await db.prepare( + "get", + `select "videoDuration", "videoID", "cid", "service" from "sponsorTimes" where "UUID" = ?`, + [UUID] + ); + let apiVideoDetails: VideoDetail = null; if (service == Service.YouTube) { // don't use cache since we have no information about the video length apiVideoDetails = await getVideoDetails(videoID, true); } - const apiVideoDuration = apiVideoDetails?.duration as VideoDuration; + const apiVideoDuration = apiVideoDetails?.page.filter((p) => p.cid == cid)[0].duration as VideoDuration; if (videoDurationChanged(videoDuration, apiVideoDuration)) { Logger.info(`Video duration changed for ${videoID} from ${videoDuration} to ${apiVideoDuration}`); await db.prepare("run", `UPDATE "sponsorTimes" SET "videoDuration" = ? WHERE "UUID" = ?`, [apiVideoDuration, UUID]); @@ -72,44 +89,57 @@ async function updateSegmentVideoDuration(UUID: SegmentUUID) { } async function checkVideoDuration(UUID: SegmentUUID) { - const { videoID, service } = await db.prepare("get", `select "videoID", "service" from "sponsorTimes" where "UUID" = ?`, [UUID]); - let apiVideoDetails: videoDetails = null; + const { videoID, cid, service } = await db.prepare("get", `select "videoID", "cid", "service" from "sponsorTimes" where "UUID" = ?`, [ + UUID, + ]); + let apiVideoDetails: VideoDetail = null; if (service == Service.YouTube) { // don't use cache since we have no information about the video length apiVideoDetails = await getVideoDetails(videoID, true); } - const apiVideoDuration = apiVideoDetails?.duration as VideoDuration; + const apiVideoDuration = apiVideoDetails?.page.filter((p) => p.cid == cid)[0].duration as VideoDuration; // if no videoDuration return early if (isNaN(apiVideoDuration)) return; // fetch latest submission - const latestSubmission = await db.prepare("get", `SELECT "videoDuration", "UUID", "timeSubmitted" + const latestSubmission = (await db.prepare( + "get", + `SELECT "videoDuration", "UUID", "timeSubmitted" FROM "sponsorTimes" - WHERE "videoID" = ? AND "service" = ? AND - "hidden" = 0 AND "shadowHidden" = 0 AND + WHERE "videoID" = ? AND "service" = ? AND + "hidden" = 0 AND "shadowHidden" = 0 AND "actionType" != 'full' AND "votes" > -2 AND "videoDuration" != 0 ORDER BY "timeSubmitted" DESC LIMIT 1`, - [videoID, service]) as {videoDuration: VideoDuration, UUID: SegmentUUID, timeSubmitted: number}; + [videoID, service] + )) as { videoDuration: VideoDuration; UUID: SegmentUUID; timeSubmitted: number }; if (latestSubmission && videoDurationChanged(latestSubmission.videoDuration, apiVideoDuration)) { Logger.info(`Video duration changed for ${videoID} from ${latestSubmission.videoDuration} to ${apiVideoDuration}`); - await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = 1 + await db.prepare( + "run", + `UPDATE "sponsorTimes" SET "hidden" = 1 WHERE "videoID" = ? AND "service" = ? AND "timeSubmitted" <= ? - AND "hidden" = 0 AND "shadowHidden" = 0 AND + AND "hidden" = 0 AND "shadowHidden" = 0 AND "actionType" != 'full' AND "votes" > -2`, - [videoID, service, latestSubmission.timeSubmitted]); + [videoID, service, latestSubmission.timeSubmitted] + ); deleteLockCategories(videoID, null, null, service).catch((e) => Logger.error(`delete lock categories after vote: ${e}`)); } } async function sendWebhooks(voteData: VoteData) { - const submissionInfoRow = await db.prepare("get", `SELECT "s"."videoID", "s"."userID", s."startTime", s."endTime", s."category", u."userName", + const submissionInfoRow = await db.prepare( + "get", + `SELECT "s"."videoID", "s"."userID", s."startTime", s."endTime", s."category", u."userName", (select count(1) from "sponsorTimes" where "userID" = s."userID") count, (select count(1) from "sponsorTimes" where "userID" = s."userID" and votes <= -2) disregarded FROM "sponsorTimes" s left join "userNames" u on s."userID" = u."userID" where s."UUID"=?`, - [voteData.UUID]); + [voteData.UUID] + ); - const userSubmissionCountRow = await db.prepare("get", `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [voteData.nonAnonUserID]); + const userSubmissionCountRow = await db.prepare("get", `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [ + voteData.nonAnonUserID, + ]); if (submissionInfoRow !== undefined && userSubmissionCountRow != undefined) { let webhookURL: string = null; @@ -134,44 +164,53 @@ async function sendWebhooks(voteData: VoteData) { const isUpvote = voteData.incrementAmount > 0; // Send custom webhooks dispatchEvent(isUpvote ? "vote.up" : "vote.down", { - "user": { - "status": getVoteAuthorRaw(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission), + user: { + status: getVoteAuthorRaw( + userSubmissionCountRow.submissionCount, + voteData.isTempVIP, + voteData.isVIP, + voteData.isOwnSubmission + ), }, - "video": { - "id": submissionInfoRow.videoID, - "title": data?.title, - "url": `https://www.youtube.com/watch?v=${videoID}`, - "thumbnail": getMaxResThumbnail(videoID), + video: { + id: submissionInfoRow.videoID, + title: data?.title, + url: `https://www.youtube.com/watch?v=${videoID}`, + thumbnail: getMaxResThumbnail(videoID), }, - "submission": { - "UUID": voteData.UUID, - "views": voteData.row.views, - "category": voteData.category, - "startTime": submissionInfoRow.startTime, - "endTime": submissionInfoRow.endTime, - "user": { - "UUID": submissionInfoRow.userID, - "username": submissionInfoRow.userName, - "submissions": { - "total": submissionInfoRow.count, - "ignored": submissionInfoRow.disregarded, + submission: { + UUID: voteData.UUID, + views: voteData.row.views, + category: voteData.category, + startTime: submissionInfoRow.startTime, + endTime: submissionInfoRow.endTime, + user: { + UUID: submissionInfoRow.userID, + username: submissionInfoRow.userName, + submissions: { + total: submissionInfoRow.count, + ignored: submissionInfoRow.disregarded, }, }, }, - "votes": { - "before": voteData.row.votes, - "after": (voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount), + votes: { + before: voteData.row.votes, + after: voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount, }, }); // Send discord message if (webhookURL !== null && !isUpvote) { - axios.post(webhookURL, { - "embeds": [{ - "title": data?.title, - "url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${(submissionInfoRow.startTime.toFixed(0) - 2)}s#requiredSegment=${voteData.UUID}`, - "description": `**${voteData.row.votes} Votes Prior | \ - ${(voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount)} Votes Now | ${voteData.row.views} \ + axios + .post(webhookURL, { + embeds: [ + { + title: data?.title, + url: `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}&t=${ + submissionInfoRow.startTime.toFixed(0) - 2 + }s#requiredSegment=${voteData.UUID}`, + description: `**${voteData.row.votes} Votes Prior | \ + ${voteData.row.votes + voteData.incrementAmount - voteData.oldIncrementAmount} Votes Now | ${voteData.row.views} \ Views**\n\n**Locked**: ${voteData.row.locked}\n\n**Submission ID:** ${voteData.UUID}\ \n**Category:** ${submissionInfoRow.category}\ \n\n**Submitted by:** ${submissionInfoRow.userName}\n${submissionInfoRow.userID}\ @@ -179,25 +218,32 @@ async function sendWebhooks(voteData: VoteData) { \n**Ignored User Submissions:** ${submissionInfoRow.disregarded}\ \n\n**Timestamp:** \ ${getFormattedTime(submissionInfoRow.startTime)} to ${getFormattedTime(submissionInfoRow.endTime)}`, - "color": 10813440, - "author": { - "name": voteData.finalResponse?.webhookMessage ?? - voteData.finalResponse?.finalMessage ?? - `${getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission)}${voteData.row.locked ? " (Locked)" : ""}`, - }, - "thumbnail": { - "url": getMaxResThumbnail(videoID), - }, - }], - }) - .then(res => { + color: 10813440, + author: { + name: + voteData.finalResponse?.webhookMessage ?? + voteData.finalResponse?.finalMessage ?? + `${getVoteAuthor( + userSubmissionCountRow.submissionCount, + voteData.isTempVIP, + voteData.isVIP, + voteData.isOwnSubmission + )}${voteData.row.locked ? " (Locked)" : ""}`, + }, + thumbnail: { + url: getMaxResThumbnail(videoID), + }, + }, + ], + }) + .then((res) => { if (res.status >= 400) { Logger.error("Error sending reported submission Discord hook"); - Logger.error(JSON.stringify((res.data))); + Logger.error(JSON.stringify(res.data)); Logger.error("\n"); } }) - .catch(err => { + .catch((err) => { Logger.error("Failed to send reported submission Discord hook."); Logger.error(JSON.stringify(err)); Logger.error("\n"); @@ -206,18 +252,43 @@ async function sendWebhooks(voteData: VoteData) { } } -async function categoryVote(UUID: SegmentUUID, userID: HashedUserID, isVIP: boolean, isTempVIP: boolean, isOwnSubmission: boolean, category: Category - , hashedIP: HashedIP, finalResponse: FinalResponse): Promise<{ status: number, message?: string }> { +async function categoryVote( + UUID: SegmentUUID, + userID: HashedUserID, + isVIP: boolean, + isTempVIP: boolean, + isOwnSubmission: boolean, + category: Category, + hashedIP: HashedIP, + finalResponse: FinalResponse +): Promise<{ status: number; message?: string }> { // Check if they've already made a vote - const usersLastVoteInfo = await privateDB.prepare("get", `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, [UUID, userID], { useReplica: true }); + const usersLastVoteInfo = await privateDB.prepare( + "get", + `select count(*) as votes, category from "categoryVotes" where "UUID" = ? and "userID" = ? group by category`, + [UUID, userID], + { useReplica: true } + ); if (usersLastVoteInfo?.category === category) { // Double vote, ignore return { status: finalResponse.finalStatus }; } - const segmentInfo = (await db.prepare("get", `SELECT "category", "actionType", "videoID", "hashedVideoID", "service", "userID", "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, - [UUID], { useReplica: true })) as {category: Category, actionType: ActionType, videoID: VideoID, hashedVideoID: VideoIDHash, service: Service, userID: UserID, locked: number}; + const segmentInfo = (await db.prepare( + "get", + `SELECT "category", "actionType", "videoID", "hashedVideoID", "service", "userID", "locked" FROM "sponsorTimes" WHERE "UUID" = ?`, + [UUID], + { useReplica: true } + )) as { + category: Category; + actionType: ActionType; + videoID: VideoID; + hashedVideoID: VideoIDHash; + service: Service; + userID: UserID; + locked: number; + }; if (!config.categorySupport[category]?.includes(segmentInfo.actionType) || segmentInfo.actionType === ActionType.Full) { return { status: 400, message: `Not allowed to change to ${category} when for segment of type ${segmentInfo.actionType}` }; @@ -227,7 +298,12 @@ async function categoryVote(UUID: SegmentUUID, userID: HashedUserID, isVIP: bool } // Ignore vote if the next category is locked - const nextCategoryLocked = await db.prepare("get", `SELECT "videoID", "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?`, [segmentInfo.videoID, segmentInfo.service, category], { useReplica: true }); + const nextCategoryLocked = await db.prepare( + "get", + `SELECT "videoID", "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ?`, + [segmentInfo.videoID, segmentInfo.service, category], + { useReplica: true } + ); if (nextCategoryLocked && !isVIP) { return { status: 200 }; } @@ -237,38 +313,76 @@ async function categoryVote(UUID: SegmentUUID, userID: HashedUserID, isVIP: bool return { status: 200 }; } - const nextCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category], { useReplica: true }); + const nextCategoryInfo = await db.prepare( + "get", + `select votes from "categoryVotes" where "UUID" = ? and category = ?`, + [UUID, category], + { useReplica: true } + ); const timeSubmitted = Date.now(); - const voteAmount = (isVIP || isTempVIP) ? 500 : 1; + const voteAmount = isVIP || isTempVIP ? 500 : 1; const ableToVote = finalResponse.finalStatus === 200; // ban status checks handled by vote() (caller function) if (ableToVote) { // Add the vote - if ((await db.prepare("get", `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])).count > 0) { + if ( + (await db.prepare("get", `select count(*) as count from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, category])) + .count > 0 + ) { // Update the already existing db entry - await db.prepare("run", `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, category]); + await db.prepare("run", `update "categoryVotes" set "votes" = "votes" + ? where "UUID" = ? and "category" = ?`, [ + voteAmount, + UUID, + category, + ]); } else { // Add a db entry - await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, category, voteAmount]); + await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [ + UUID, + category, + voteAmount, + ]); } // Add the info into the private db if (usersLastVoteInfo?.votes > 0) { // Reverse the previous vote - await db.prepare("run", `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [voteAmount, UUID, usersLastVoteInfo.category]); - - await privateDB.prepare("run", `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, [category, timeSubmitted, hashedIP, userID, UUID]); + await db.prepare("run", `update "categoryVotes" set "votes" = "votes" - ? where "UUID" = ? and "category" = ?`, [ + voteAmount, + UUID, + usersLastVoteInfo.category, + ]); + + await privateDB.prepare( + "run", + `update "categoryVotes" set "category" = ?, "timeSubmitted" = ?, "hashedIP" = ? where "userID" = ? and "UUID" = ?`, + [category, timeSubmitted, hashedIP, userID, UUID] + ); } else { - await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, category, timeSubmitted]); + await privateDB.prepare( + "run", + `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, + [UUID, userID, hashedIP, category, timeSubmitted] + ); } // See if the submissions category is ready to change - const currentCategoryInfo = await db.prepare("get", `select votes from "categoryVotes" where "UUID" = ? and category = ?`, [UUID, segmentInfo.category], { useReplica: true }); + const currentCategoryInfo = await db.prepare( + "get", + `select votes from "categoryVotes" where "UUID" = ? and category = ?`, + [UUID, segmentInfo.category], + { useReplica: true } + ); - const submissionInfo = await db.prepare("get", `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, [UUID], { useReplica: true }); - const isSubmissionVIP = submissionInfo && await isUserVIP(submissionInfo.userID); + const submissionInfo = await db.prepare( + "get", + `SELECT "userID", "timeSubmitted", "votes" FROM "sponsorTimes" WHERE "UUID" = ?`, + [UUID], + { useReplica: true } + ); + const isSubmissionVIP = submissionInfo && (await isUserVIP(submissionInfo.userID)); const startingVotes = isSubmissionVIP ? 10000 : 1; // Change this value from 1 in the future to make it harder to change categories @@ -277,15 +391,28 @@ async function categoryVote(UUID: SegmentUUID, userID: HashedUserID, isVIP: bool // Add submission as vote if (!currentCategoryInfo && submissionInfo) { - await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [UUID, segmentInfo.category, currentCategoryCount]); - await privateDB.prepare("run", `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, [UUID, submissionInfo.userID, "unknown", segmentInfo.category, submissionInfo.timeSubmitted]); + await db.prepare("run", `insert into "categoryVotes" ("UUID", "category", "votes") values (?, ?, ?)`, [ + UUID, + segmentInfo.category, + currentCategoryCount, + ]); + await privateDB.prepare( + "run", + `insert into "categoryVotes" ("UUID", "userID", "hashedIP", "category", "timeSubmitted") values (?, ?, ?, ?, ?)`, + [UUID, submissionInfo.userID, "unknown", segmentInfo.category, submissionInfo.timeSubmitted] + ); } const nextCategoryCount = (nextCategoryInfo?.votes || 0) + voteAmount; //TODO: In the future, raise this number from zero to make it harder to change categories // VIPs change it every time - if (isVIP || isTempVIP || isOwnSubmission || nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2)) { + if ( + isVIP || + isTempVIP || + isOwnSubmission || + nextCategoryCount - currentCategoryCount >= Math.max(Math.ceil(submissionInfo?.votes / 2), 2) + ) { // Replace the category await db.prepare("run", `update "sponsorTimes" set "category" = ? where "UUID" = ?`, [category, UUID]); } @@ -317,7 +444,13 @@ export async function voteOnSponsorTime(req: Request, res: Response): Promise { +export async function vote( + ip: IPAddress, + UUID: SegmentUUID, + paramUserID: UserID, + type: number, + category?: Category +): Promise<{ status: number; message?: string; json?: unknown }> { // missing key parameters if (!UUID || !paramUserID || !(type !== undefined || category)) { return { status: 400 }; @@ -347,7 +480,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID finalStatus: 200, finalMessage: null, webhookType: VoteWebhookType.Normal, - webhookMessage: null + webhookMessage: null, }; const segmentInfo: DBSegment = await db.prepare("get", `SELECT * from "sponsorTimes" WHERE "UUID" = ?`, [UUID]); @@ -372,16 +505,22 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID const MILLISECONDS_IN_HOUR = 3600000; const now = Date.now(); - const warnings = (await db.prepare("all", `SELECT "reason" FROM warnings WHERE "userID" = ? AND "issueTime" > ? AND enabled = 1 AND type = 0`, - [nonAnonUserID, Math.floor(now - (config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR))], - )); + const warnings = await db.prepare( + "all", + `SELECT "reason" FROM warnings WHERE "userID" = ? AND "issueTime" > ? AND enabled = 1 AND type = 0`, + [nonAnonUserID, Math.floor(now - config.hoursAfterWarningExpires * MILLISECONDS_IN_HOUR)] + ); if (warnings.length >= config.maxNumberOfActiveWarnings) { const warningReason = warnings[0]?.reason; lock.unlock(); - return { status: 403, message: "Vote rejected due to a tip from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. " + + return { + status: 403, + message: + "Vote rejected due to a tip from a moderator. This means that we noticed you were making some common mistakes that are not malicious, and we just want to clarify the rules. " + "Could you please send a message in Discord or Matrix so we can further help you?" + - `${(warningReason.length > 0 ? ` Tip message: '${warningReason}'` : "")}` }; + `${warningReason.length > 0 ? ` Tip message: '${warningReason}'` : ""}`, + }; } // we can return out of the function early if the user is banned after warning checks @@ -402,10 +541,15 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID // If not upvote, or an upvote on a dead segment (for ActionType.Full) if (!isVIP && (type != 1 || segmentInfo.votes <= -2)) { const isSegmentLocked = segmentInfo.locked; - const isVideoLocked = async () => !!(await db.prepare("get", `SELECT "category" FROM "lockCategories" WHERE + const isVideoLocked = async () => + !!(await db.prepare( + "get", + `SELECT "category" FROM "lockCategories" WHERE "videoID" = ? AND "service" = ? AND "category" = ? AND "actionType" = ?`, - [segmentInfo.videoID, segmentInfo.service, segmentInfo.category, segmentInfo.actionType], { useReplica: true })); - if (isSegmentLocked || await isVideoLocked()) { + [segmentInfo.videoID, segmentInfo.service, segmentInfo.category, segmentInfo.actionType], + { useReplica: true } + )); + if (isSegmentLocked || (await isVideoLocked())) { finalResponse.blockVote = true; finalResponse.webhookType = VoteWebhookType.Rejected; finalResponse.webhookMessage = "Vote rejected: A moderator has decided that this segment is correct"; @@ -413,8 +557,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID } // if on downvoted non-full segment and is not VIP/ tempVIP/ submitter - if (!isNaN(type) && segmentInfo.votes <= -2 && segmentInfo.actionType !== ActionType.Full && - !(isVIP || isTempVIP || isOwnSubmission)) { + if (!isNaN(type) && segmentInfo.votes <= -2 && segmentInfo.actionType !== ActionType.Full && !(isVIP || isTempVIP || isOwnSubmission)) { if (type == 1) { lock.unlock(); return { status: 403, message: "Not allowed to upvote segment with too many downvotes unless you are VIP." }; @@ -426,7 +569,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID } } - const voteTypeEnum = (type == 0 || type == 1 || type == 20) ? voteTypes.normal : voteTypes.incorrect; + const voteTypeEnum = type == 0 || type == 1 || type == 20 ? voteTypes.normal : voteTypes.incorrect; // no restrictions on checkDuration // check duration of all submissions on this video @@ -436,7 +579,9 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID try { // check if vote has already happened - const votesRow = await privateDB.prepare("get", `SELECT "type" FROM "votes" WHERE "userID" = ? AND "UUID" = ?`, [userID, UUID], { useReplica: true }); + const votesRow = await privateDB.prepare("get", `SELECT "type" FROM "votes" WHERE "userID" = ? AND "UUID" = ?`, [userID, UUID], { + useReplica: true, + }); // -1 for downvote, 1 for upvote. Maybe more depending on reputation in the future // oldIncrementAmount will be zero if row is null @@ -493,25 +638,48 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID // Only change the database if they have made a submission before and haven't voted recently // ban status check was handled earlier (w/ early return) - const ableToVote = isVIP || isTempVIP || ( - (!(isOwnSubmission && incrementAmount > 0 && oldIncrementAmount >= 0) - && !(originalType === VoteType.Malicious && segmentInfo.actionType !== ActionType.Chapter) - && !finalResponse.blockVote - && finalResponse.finalStatus === 200 - && (await db.prepare("get", `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ? AND "category" = ? AND "votes" > -2 AND "hidden" = 0 AND "shadowHidden" = 0 LIMIT 1`, [nonAnonUserID, segmentInfo.category], { useReplica: true }) !== undefined) - && (await privateDB.prepare("get", `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, [UUID, hashedIP, userID], { useReplica: true })) === undefined) - ); + const ableToVote = + isVIP || + isTempVIP || + (!(isOwnSubmission && incrementAmount > 0 && oldIncrementAmount >= 0) && + !(originalType === VoteType.Malicious && segmentInfo.actionType !== ActionType.Chapter) && + !finalResponse.blockVote && + finalResponse.finalStatus === 200 && + (await db.prepare( + "get", + `SELECT "userID" FROM "sponsorTimes" WHERE "userID" = ? AND "category" = ? AND "votes" > -2 AND "hidden" = 0 AND "shadowHidden" = 0 LIMIT 1`, + [nonAnonUserID, segmentInfo.category], + { useReplica: true } + )) !== undefined && + (await privateDB.prepare( + "get", + `SELECT "UUID" FROM "votes" WHERE "UUID" = ? AND "hashedIP" = ? AND "userID" != ?`, + [UUID, hashedIP, userID], + { useReplica: true } + )) === undefined); if (ableToVote) { //update the votes table if (votesRow) { - await privateDB.prepare("run", `UPDATE "votes" SET "type" = ?, "originalType" = ? WHERE "userID" = ? AND "UUID" = ?`, [type, originalType, userID, UUID]); + await privateDB.prepare("run", `UPDATE "votes" SET "type" = ?, "originalType" = ? WHERE "userID" = ? AND "UUID" = ?`, [ + type, + originalType, + userID, + UUID, + ]); } else { - await privateDB.prepare("run", `INSERT INTO "votes" ("UUID", "userID", "hashedIP", "type", "normalUserID", "originalType") VALUES(?, ?, ?, ?, ?, ?)`, [UUID, userID, hashedIP, type, nonAnonUserID, originalType]); + await privateDB.prepare( + "run", + `INSERT INTO "votes" ("UUID", "userID", "hashedIP", "type", "normalUserID", "originalType") VALUES(?, ?, ?, ?, ?, ?)`, + [UUID, userID, hashedIP, type, nonAnonUserID, originalType] + ); } // update the vote count on this sponsorTime - await db.prepare("run", `UPDATE "sponsorTimes" SET "votes" = "votes" + ? WHERE "UUID" = ?`, [incrementAmount - oldIncrementAmount, UUID]); + await db.prepare("run", `UPDATE "sponsorTimes" SET "votes" = "votes" + ? WHERE "UUID" = ?`, [ + incrementAmount - oldIncrementAmount, + UUID, + ]); // tempVIP can bring back hidden segments if (isTempVIP && incrementAmount > 0 && voteTypeEnum === voteTypes.normal) { @@ -523,8 +691,10 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID // Update video duration in case that caused it to be hidden await updateSegmentVideoDuration(UUID); // unhide & unlock - await db.prepare("run", 'UPDATE "sponsorTimes" SET "locked" = 1, "hidden" = 0, "shadowHidden" = 0 WHERE "UUID" = ?', [UUID]); - // on VIP downvote/ undovote, also unlock submission + await db.prepare("run", 'UPDATE "sponsorTimes" SET "locked" = 1, "hidden" = 0, "shadowHidden" = 0 WHERE "UUID" = ?', [ + UUID, + ]); + // on VIP downvote/ undovote, also unlock submission } else if (isVIP && incrementAmount <= 0 && voteTypeEnum === voteTypes.normal) { await db.prepare("run", 'UPDATE "sponsorTimes" SET "locked" = 0 WHERE "UUID" = ?', [UUID]); } @@ -544,7 +714,7 @@ export async function vote(ip: IPAddress, UUID: SegmentUUID, paramUserID: UserID category, incrementAmount, oldIncrementAmount, - finalResponse + finalResponse, }).catch((e) => Logger.error(`Sending vote webhook: ${e}`)); } diff --git a/src/types/portVideo.model.ts b/src/types/portVideo.model.ts index 209aed3e..668d83fe 100644 --- a/src/types/portVideo.model.ts +++ b/src/types/portVideo.model.ts @@ -17,6 +17,7 @@ export interface PortVideoInterface { export interface PortVideo { bvID: VideoID; + cid: string; ytbID: VideoID; UUID: portVideoUUID; votes: number; diff --git a/src/types/segments.model.ts b/src/types/segments.model.ts index 5b657315..851558d0 100644 --- a/src/types/segments.model.ts +++ b/src/types/segments.model.ts @@ -46,7 +46,17 @@ export interface IncomingSegment { ignoreSegment?: boolean; } +export interface VideoLabel { + cid: string; + category: Category; + UUID: SegmentUUID; + videoDuration: VideoDuration; + locked: boolean; + votes: number; +} + export interface Segment { + cid: string; category: Category; actionType: ActionType; segment: number[]; @@ -65,6 +75,7 @@ export enum Visibility { export interface DBSegment { videoID: VideoID; + cid: string; startTime: number; endTime: number; @@ -112,6 +123,10 @@ export interface VotableObjectWithWeight extends VotableObject { weight: number; } +export interface VideoLabelData { + segments: VideoLabel[]; +} + export interface VideoData { segments: Segment[]; } diff --git a/src/utils/bilibiliApi.ts b/src/utils/bilibiliApi.ts index d26e9ad3..a41adf74 100644 --- a/src/utils/bilibiliApi.ts +++ b/src/utils/bilibiliApi.ts @@ -28,11 +28,9 @@ export class BilibiliAPI { } static async getVideoDetailView(videoID: string): Promise { - // TODO: validate video id - Logger.info(`Getting video detail from view API: ${videoID}`); const url = "https://api.bilibili.com/x/web-interface/view"; - const result = await axios.get(url, { params: { bvid: videoID }, timeout: 3500 }); + const result = await axios.get(url, { params: { bvid: videoID }, timeout: 20000 }); if (result.status === 200 && result.data.code === 0) { return result.data.data; diff --git a/src/utils/getVideoDetails.ts b/src/utils/getVideoDetails.ts index 3c4afece..1705f6fd 100644 --- a/src/utils/getVideoDetails.ts +++ b/src/utils/getVideoDetails.ts @@ -4,32 +4,41 @@ import { Logger } from "./logger"; import { QueryCacher } from "./queryCacher"; import { videoDetailCacheKey } from "./redisKeys"; -export interface videoDetails { - videoId: string; +export interface VideoPageDetail { + cid: string; + page: number; + part: string; duration: number; +} + +export interface VideoDetail { + videoId: string; authorId: string; authorName: string; title: string; published: number; + page: VideoPageDetail[]; } -const convertFromVideoViewAPI = (videoId: string, input: BilibiliVideoDetailView): videoDetails => ({ - videoId: videoId, - duration: input.pages.length >= 1 && input.pages[0].duration ? input.pages[0].duration : input.duration, - authorId: input.owner.mid.toString(), - authorName: input.owner.name, - title: input.title, - published: input.pubdate, -}); +const convertFromVideoViewAPI = (videoId: string, input: BilibiliVideoDetailView): VideoDetail => { + return { + videoId: videoId, + authorId: input.owner.mid.toString(), + authorName: input.owner.name, + title: input.title, + published: input.pubdate, + page: input.pages.map((page) => ({ cid: `${page.cid}`, page: page.page, part: page.part, duration: page.duration })), + }; +}; -export function getVideoDetails(videoId: string, ignoreCache = false): Promise { +export function getVideoDetails(videoId: string, ignoreCache = false): Promise { if (ignoreCache) { QueryCacher.clearKey(videoDetailCacheKey(videoId)); } - return QueryCacher.get(() => getVideoDetailsFromAPI(videoId), videoDetailCacheKey(videoId)); + return QueryCacher.get(() => getVideoDetailsFromAPI(videoId), videoDetailCacheKey(videoId), 365 * 24 * 60 * 60); - async function getVideoDetailsFromAPI(videoId: string): Promise { + async function getVideoDetailsFromAPI(videoId: string): Promise { try { const data = await BilibiliAPI.getVideoDetailView(videoId); return convertFromVideoViewAPI(videoId, data); diff --git a/src/utils/queryCacher.ts b/src/utils/queryCacher.ts index f624bec0..5cb4fb37 100644 --- a/src/utils/queryCacher.ts +++ b/src/utils/queryCacher.ts @@ -149,6 +149,10 @@ function clearKey(key: string): void { redis.del(key).catch((err) => Logger.error(err)); } +function clearKeyPattern(keyPattern: string): void { + redis.delPattern(keyPattern).catch((err) => Logger.error(err)); +} + function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoIDHash; @@ -157,7 +161,7 @@ function clearSegmentCache(videoInfo: { }): void { if (videoInfo) { redis.del(skipSegmentsKey(videoInfo.videoID, videoInfo.service)).catch((err) => Logger.error(err)); - redis.del(skipSegmentGroupsKey(videoInfo.videoID, videoInfo.service)).catch((err) => Logger.error(err)); + clearKeyPattern(skipSegmentGroupsKey(videoInfo.videoID, "*", videoInfo.service)); redis.del(skipSegmentsHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(videoLabelsKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); redis.del(videoLabelsHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err)); @@ -220,6 +224,7 @@ export const QueryCacher = { getTraced, getAndSplit, clearKey, + clearKeyPattern, clearSegmentCache, clearSegmentCacheByID, getKeyLastModified, diff --git a/src/utils/redis.ts b/src/utils/redis.ts index 05c1b999..9b221a13 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -1,13 +1,14 @@ -import { config } from "../config"; -import { Logger } from "./logger"; -import { RedisClientType, SetOptions, createClient } from "redis"; -import { RedisCommandArgument, RedisCommandArguments, RedisCommandRawReply } from "@redis/client/dist/lib/commands"; import { RedisClientOptions } from "@redis/client/dist/lib/client"; +import { RedisCommandArgument, RedisCommandArguments, RedisCommandRawReply } from "@redis/client/dist/lib/commands"; +import { ScanCommandOptions } from "@redis/client/dist/lib/commands/SCAN"; +import { LRUCache } from "lru-cache"; +import { compress, uncompress } from "lz4-napi"; import { RedisReply } from "rate-limit-redis"; +import { RedisClientType, SetOptions, createClient } from "redis"; +import { config } from "../config"; import { db } from "../databases/databases"; import { Postgres } from "../databases/Postgres"; -import { compress, uncompress } from "lz4-napi"; -import { LRUCache } from "lru-cache"; +import { Logger } from "./logger"; import { shouldClientCacheKey } from "./redisKeys"; export interface RedisStats { @@ -31,6 +32,7 @@ interface RedisSB { setEx(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; setExWithCache(key: RedisCommandArgument, seconds: number, value: RedisCommandArgument): Promise; del(...keys: [RedisCommandArgument]): Promise; + delPattern(pattern: RedisCommandArgument): Promise; increment?(key: RedisCommandArgument): Promise; sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise; ttl(key: RedisCommandArgument): Promise; @@ -45,6 +47,7 @@ let exportClient: RedisSB = { setEx: () => Promise.resolve(null), setExWithCache: () => Promise.resolve(null), del: () => Promise.resolve(null), + delPattern: () => Promise.resolve(null), increment: () => Promise.resolve(null), sendCommand: () => Promise.resolve(null), quit: () => Promise.resolve(null), @@ -199,6 +202,13 @@ if (config.redis?.enabled) { } }; + const scan = client.scan.bind(client); + exportClient.delPattern = async (pattern) => { + const keys = await scan(0, { MATCH: pattern } as ScanCommandOptions); + await Promise.allSettled(keys.keys.map((key) => exportClient.del(key))); + return keys.keys.length; + }; + const ttl = client.ttl.bind(client); exportClient.ttl = async (key) => { if (cache && cacheClient && ttlCache.has(key)) { diff --git a/src/utils/redisKeys.ts b/src/utils/redisKeys.ts index b8c85e79..98bd7d4a 100644 --- a/src/utils/redisKeys.ts +++ b/src/utils/redisKeys.ts @@ -4,24 +4,32 @@ import { HashedValue } from "../types/hash.model"; import { Logger } from "./logger"; import { RedisCommandArgument } from "@redis/client/dist/lib/commands"; -export const skipSegmentsKey = (videoID: VideoID, service: Service): string => - `segments.v4.${service}.videoID.${videoID}`; +export function skipSegmentsKey(videoID: VideoID, service: Service): string { + return `segments.v5.${service}.videoID.${videoID}`; +} -export const skipSegmentGroupsKey = (videoID: VideoID, service: Service): string => - `segments.groups.v3.${service}.videoID.${videoID}`; +export function skipSegmentGroupsKey(videoID: VideoID, cid: string, service: Service): string { + if (cid === "*") { + return `segments.groups.v4.${service}.videoID.${videoID}*`; + } + return `segments.groups.v4.${service}.videoID.${videoID}.${cid}`; +} export function skipSegmentsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 4) as VideoIDHash; if (hashedVideoIDPrefix.length !== 4) Logger.warn(`Redis skip segment hash-prefix key is not length 4! ${hashedVideoIDPrefix}`); - return `segments.v4.${service}.${hashedVideoIDPrefix}`; + return `segments.v5.${service}.${hashedVideoIDPrefix}`; +} + +export function cidListKey(videoID: VideoID): string { + return `cid.videoID.${videoID}`; } export const shadowHiddenIPKey = (videoID: VideoID, timeSubmitted: number, service: Service): string => `segments.v1.${service}.videoID.${videoID}.shadow.${timeSubmitted}`; -export const reputationKey = (userID: UserID): string => - `reputation.v1.user.${userID}`; +export const reputationKey = (userID: UserID): string => `reputation.v1.user.${userID}`; export function ratingHashKey(hashPrefix: VideoIDHash, service: Service): string { hashPrefix = hashPrefix.substring(0, 4) as VideoIDHash; @@ -36,11 +44,9 @@ export function shaHashKey(singleIter: HashedValue): string { return `sha.hash.${singleIter}`; } -export const tempVIPKey = (userID: HashedUserID): string => - `vip.temp.${userID}`; +export const tempVIPKey = (userID: HashedUserID): string => `vip.temp.${userID}`; -export const videoLabelsKey = (videoID: VideoID, service: Service): string => - `labels.v1.${service}.videoID.${videoID}`; +export const videoLabelsKey = (videoID: VideoID, service: Service): string => `labels.v1.${service}.videoID.${videoID}`; export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Service): string { hashedVideoIDPrefix = hashedVideoIDPrefix.substring(0, 3) as VideoIDHash; @@ -49,7 +55,7 @@ export function videoLabelsHashKey(hashedVideoIDPrefix: VideoIDHash, service: Se return `labels.v1.${service}.${hashedVideoIDPrefix}`; } -export function userFeatureKey (userID: HashedUserID, feature: Feature): string { +export function userFeatureKey(userID: HashedUserID, feature: Feature): string { return `user.v1.${userID}.feature.${feature}`; } @@ -78,5 +84,5 @@ export function portVideoUserCountKey() { } export function videoDetailCacheKey(videoID: string) { - return `video.detail.v1.videoID.${videoID}`; -} \ No newline at end of file + return `video.detail.v2.videoID.${videoID}`; +} diff --git a/src/utils/timeUtil.ts b/src/utils/timeUtil.ts new file mode 100644 index 00000000..dffb47d6 --- /dev/null +++ b/src/utils/timeUtil.ts @@ -0,0 +1,3 @@ +export function sleep(time: number) { + return new Promise((resolve) => setTimeout(resolve, time)); +} diff --git a/src/utils/bilibiliID.ts b/src/validate/bilibiliID.ts similarity index 100% rename from src/utils/bilibiliID.ts rename to src/validate/bilibiliID.ts diff --git a/src/validate/validator.ts b/src/validate/validator.ts new file mode 100644 index 00000000..d39bac29 --- /dev/null +++ b/src/validate/validator.ts @@ -0,0 +1,32 @@ +import { config } from "../config"; + +interface ValidationResult { + pass: boolean; + errorMessage?: string; +} + +function pass(): ValidationResult { + return { pass: true }; +} + +function fail(errorMessage: string): ValidationResult { + return { pass: false, errorMessage: errorMessage }; +} + +export function validateCid(cid: string) { + if (!cid || /^[1-9]\d*$/.test(cid)) { + return pass(); + } + return fail("cid有误"); +} + +export function validatePrivateUserID(userID: string): ValidationResult { + const minLength = config.minUserIDLength; + if (typeof userID !== "string") { + return fail("私人用户ID有误"); + } + if (userID?.length < minLength) { + return fail(`私人用户ID至少 ${minLength} 个字符长`); + } + return pass(); +} diff --git a/src/utils/youtubeID.ts b/src/validate/youtubeID.ts similarity index 100% rename from src/utils/youtubeID.ts rename to src/validate/youtubeID.ts