From bf4b330cba2a3989102f6b146ea07abfcee5f996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=A1ko=20Hevery?= Date: Tue, 15 Oct 2024 21:24:16 -0400 Subject: [PATCH 1/3] fix(insights): add query limits to prevent render timeout (#6980) --- packages/insights/src/db/query.ts | 2 +- packages/insights/src/db/sql-manifest.ts | 3 +++ packages/insights/src/db/sql-routes.ts | 3 +++ packages/insights/src/db/sql-user.ts | 2 ++ .../insights/src/routes/api/v1/[publicApiKey]/post/index.tsx | 4 +++- .../src/routes/api/v1/[publicApiKey]/post/manifest/index.tsx | 1 + .../insights/src/routes/app/[publicApiKey]/errors/index.tsx | 1 + packages/insights/src/routes/app/index.tsx | 2 +- 8 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/insights/src/db/query.ts b/packages/insights/src/db/query.ts index 55308c45348..5799cd51eb5 100644 --- a/packages/insights/src/db/query.ts +++ b/packages/insights/src/db/query.ts @@ -47,7 +47,7 @@ export async function getEdges( .from(edgeTable) .where(where) .groupBy(edgeTable.from, edgeTable.to) - .limit(limit || 10_000); // TODO: The 10_000 limit is due to Turso serialization format not being efficient, upgrade this once Turso is fixed. + .limit(limit || 5_000); // TODO: The 5_000 limit is due to Turso serialization format not being efficient, upgrade this once Turso is fixed. const rows = await query.all(); return rows.map((e) => ({ from: e.from, diff --git a/packages/insights/src/db/sql-manifest.ts b/packages/insights/src/db/sql-manifest.ts index f1bcb8d9d35..7b09b32cd75 100644 --- a/packages/insights/src/db/sql-manifest.ts +++ b/packages/insights/src/db/sql-manifest.ts @@ -12,6 +12,7 @@ export async function dbGetManifests( .from(manifestTable) .where(and(eq(manifestTable.publicApiKey, publicApiKey))) .orderBy(sql`${manifestTable.timestamp} DESC`) + .limit(1000) .all(); return manifests; } @@ -37,6 +38,7 @@ export async function dbGetManifestStats( .where(and(eq(manifestTable.publicApiKey, publicApiKey))) .groupBy(manifestTable.hash) .orderBy(sql`${manifestTable.timestamp} DESC`) + .limit(1000) .all(); return manifests.map((manifest) => { return { @@ -94,6 +96,7 @@ export async function dbGetManifestHashes( .where(and(eq(manifestTable.publicApiKey, publicApiKey))) .groupBy(manifestTable.hash) .orderBy(sql`${manifestTable.timestamp} DESC`) + .limit(1000) .all(); const hashes: string[] = []; let sum = 0; diff --git a/packages/insights/src/db/sql-routes.ts b/packages/insights/src/db/sql-routes.ts index 565cd15c068..1a3162006b2 100644 --- a/packages/insights/src/db/sql-routes.ts +++ b/packages/insights/src/db/sql-routes.ts @@ -30,6 +30,7 @@ export async function getRoutes( .where(where) .groupBy(routesTable.route, routesTable.symbol) .orderBy(sql`${routesTable.route}`, desc(sumTimelineCount)) + .limit(1000) .all(); return query.map((row) => ({ route: row.route, @@ -63,6 +64,7 @@ export async function getRouteNames( .where(where) .groupBy(routesTable.route) .orderBy(sql`${routesTable.route}`) + .limit(1000) .all(); return query.map((row) => ({ route: row.route, @@ -99,6 +101,7 @@ export async function getRouteTimeline( .where(where) .groupBy(routesTable.route, routesTable.symbol) .orderBy(sql`${routesTable.route}`, desc(sumTimelineCount)) + .limit(1000) .all(); return query.map((row) => ({ route: row.route, diff --git a/packages/insights/src/db/sql-user.ts b/packages/insights/src/db/sql-user.ts index 4e1da01865b..d7f18fb985a 100644 --- a/packages/insights/src/db/sql-user.ts +++ b/packages/insights/src/db/sql-user.ts @@ -40,6 +40,7 @@ export const dbGetInsightUser = async (email: string): Promise => .leftJoin(userApplicationMap, eq(usersTable.id, userApplicationMap.userId)) .leftJoin(applicationTable, eq(applicationTable.id, userApplicationMap.applicationId)) .where(eq(usersTable.email, email)) + .limit(1000) .all(); if (users.length === 0) { const insert = await db @@ -72,6 +73,7 @@ export const dbGetUsersForApplication = async (publicApiKey: string) => { .leftJoin(applicationTable, eq(applicationTable.id, userApplicationMap.applicationId)) .where(eq(applicationTable.publicApiKey, publicApiKey)) .orderBy(usersTable.email) + .limit(1000) .all(); return users.map((user) => user.email); }; diff --git a/packages/insights/src/routes/api/v1/[publicApiKey]/post/index.tsx b/packages/insights/src/routes/api/v1/[publicApiKey]/post/index.tsx index 8d4f072787f..51e3622449c 100644 --- a/packages/insights/src/routes/api/v1/[publicApiKey]/post/index.tsx +++ b/packages/insights/src/routes/api/v1/[publicApiKey]/post/index.tsx @@ -9,7 +9,9 @@ export const onPost: RequestHandler = async ({ exit, json, request, params }) => const payloadJson = await request.json(); migrate1(payloadJson); // publicApiKey is always part of the URL as route parameter. - payloadJson.publicApiKey = params.publicApiKey; + if (!payloadJson.publicApiKey) { + payloadJson.publicApiKey = params.publicApiKey; + } const payload = InsightsPayload.parse(payloadJson); // console.log('API: POST: symbol', payload); exit(); diff --git a/packages/insights/src/routes/api/v1/[publicApiKey]/post/manifest/index.tsx b/packages/insights/src/routes/api/v1/[publicApiKey]/post/manifest/index.tsx index b8f0ec2f9dc..47a6093914b 100644 --- a/packages/insights/src/routes/api/v1/[publicApiKey]/post/manifest/index.tsx +++ b/packages/insights/src/routes/api/v1/[publicApiKey]/post/manifest/index.tsx @@ -22,6 +22,7 @@ export const onPost: RequestHandler = async ({ exit, json, request, params }) => eq(symbolDetailTable.manifestHash, manifestHash) ) ) + .limit(1000) .all(); const existingMap = new Map(); existing.forEach((row) => existingMap.set(row.hash, row)); diff --git a/packages/insights/src/routes/app/[publicApiKey]/errors/index.tsx b/packages/insights/src/routes/app/[publicApiKey]/errors/index.tsx index e1ed77aa04b..d1a030f53f0 100644 --- a/packages/insights/src/routes/app/[publicApiKey]/errors/index.tsx +++ b/packages/insights/src/routes/app/[publicApiKey]/errors/index.tsx @@ -13,6 +13,7 @@ export const useErrors = routeLoader$(async ({ params }) => { .where(eq(errorTable.publicApiKey, params.publicApiKey)) .limit(1000) .orderBy(sql`${errorTable.timestamp} DESC`) + .limit(1000) .all(); return errors; }); diff --git a/packages/insights/src/routes/app/index.tsx b/packages/insights/src/routes/app/index.tsx index e746a631c3c..410d317e5d3 100644 --- a/packages/insights/src/routes/app/index.tsx +++ b/packages/insights/src/routes/app/index.tsx @@ -21,7 +21,7 @@ export const useApps = routeLoader$(async ({ sharedMap }) => { // The user has nothing attached to it. return []; } - return query.all(); + return query.limit(1000).all(); }); export default component$(() => { From cc253b5fb6d46c8953048c1dc4e121537d977acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=A1ko=20Hevery?= Date: Tue, 15 Oct 2024 21:28:56 -0400 Subject: [PATCH 2/3] Pr insights sql fix (#6981) * fix(insights): add query limits to prevent render timeout * fix(insights): add query limits to prevent render timeout From f2acbcc659be474c1c0a14e8ce67277ed3c89ad7 Mon Sep 17 00:00:00 2001 From: Varixo Date: Wed, 16 Oct 2024 20:43:01 +0200 Subject: [PATCH 3/3] fix qwik-worker import --- packages/qwik-worker/src/worker.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/qwik-worker/src/worker.js b/packages/qwik-worker/src/worker.js index 3d358236ec8..0896b3a2359 100644 --- a/packages/qwik-worker/src/worker.js +++ b/packages/qwik-worker/src/worker.js @@ -1,4 +1,4 @@ -import { _deserializeData } from '@builder.io/qwik'; +import { _deserialize } from '@builder.io/qwik'; globalThis.document = { nodeType: 9, @@ -30,7 +30,7 @@ globalThis.onmessage = async ({ data }) => { }, }; try { - const [qrl, ...args] = _deserializeData(data[3], containerEl); + const [qrl, ...args] = _deserialize(data[3], containerEl); const output = await qrl.apply(undefined, args); self.postMessage([requestId, true, output]); } catch (err) {