feat: add pagination based on limit and offset (#162)

Also temporarily avoid time slot based approach for log drill down.
parseablehq · Nov 2, 2023 · 5e55f8e · 5e55f8e
1 parent 377328a
commit 5e55f8e
Show file tree

Hide file tree

Showing 5 changed files with 170 additions and 46 deletions.
diff --git a/src/api/query.ts b/src/api/query.ts
@@ -2,10 +2,18 @@ import { Axios } from './axios';
 import { LOG_QUERY_URL } from './constants';
 import { LogsQuery } from '@/@types/parseable/api/query';
 
-export const getQueryLogs = (logsQuery: LogsQuery) => {
-	const { startTime, endTime, streamName } = logsQuery;
+type QueryLogs = {
+	streamName: string;
+	startTime: Date;
+	endTime: Date;
+	limit: number;
+	pageOffset: number;
+};
 
-	const query = `SELECT * FROM ${streamName}`;
+export const getQueryLogs = (logsQuery: QueryLogs) => {
+	const { startTime, endTime, streamName, limit, pageOffset } = logsQuery;
+
+	const query = `SELECT * FROM ${streamName} LIMIT ${limit} OFFSET ${pageOffset}`;
 
 	return Axios().post(
 		LOG_QUERY_URL,
@@ -46,4 +54,4 @@ export const getQueryCount = (logsQuery: LogsQuery) => {
 		},
 		{},
 	);
-};
+};
diff --git a/src/hooks/useQueryLogs.ts b/src/hooks/useQueryLogs.ts
@@ -1,11 +1,19 @@
-import { SortOrder, type Log, type LogsData, type LogsQuery, type LogsSearch } from '@/@types/parseable/api/query';
+import { SortOrder, type Log, type LogsData, type LogsSearch } from '@/@types/parseable/api/query';
 import { getQueryLogs } from '@/api/query';
 import { StatusCodes } from 'http-status-codes';
 import useMountedState from './useMountedState';
 import { useCallback, useEffect, useMemo, useRef, useTransition } from 'react';
 import { LOG_QUERY_LIMITS } from '@/pages/Logs/Context';
 import { parseLogData } from '@/utils';
 
+type QueryLogs = {
+	streamName: string;
+	startTime: Date;
+	endTime: Date;
+	limit: number;
+	pageOffset: number;
+};
+
 export const useQueryLogs = () => {
 	// data ref will always have the unfiltered data.
 	// Only mutate it when data is fetched, otherwise read only
@@ -125,7 +133,7 @@ export const useQueryLogs = () => {
 		}
 	}, [data]);
 
-	const getQueryData = async (logsQuery: LogsQuery) => {
+	const getQueryData = async (logsQuery: QueryLogs) => {
 		try {
 			setLoading(true);
 			setError(null);

diff --git a/src/pages/Logs/LogTable.tsx b/src/pages/Logs/LogTable.tsx
@@ -15,6 +15,7 @@ import {
 	Button,
 	Pagination,
 	Loader,
+	Group,
 } from '@mantine/core';
 import { useCallback, useEffect, useMemo, useRef } from 'react';
 import type { FC } from 'react';
@@ -33,12 +34,14 @@ import FilterPills from './FilterPills';
 import { useHeaderContext } from '@/layouts/MainLayout/Context';
 import dayjs from 'dayjs';
 import { SortOrder } from '@/@types/parseable/api/query';
+import { usePagination } from '@mantine/hooks';
 
 const skipFields = ['p_metadata', 'p_tags'];
+const loadLimit = 9000;
 
 const LogTable: FC = () => {
 	const {
-		state: { subLogStreamError, subGapTime },
+		state: { subLogStreamError },
 	} = useLogsPageContext();
 	const {
 		state: { subLogSearch, subLogQuery, subRefreshInterval, subLogSelectedTimeRange },
@@ -48,6 +51,7 @@ const LogTable: FC = () => {
 	const [logStreamError, setLogStreamError] = useMountedState<string | null>(null);
 	const [columnToggles, setColumnToggles] = useMountedState<Map<string, boolean>>(new Map());
 	const [pinnedColumns, setPinnedColumns] = useMountedState<Set<string>>(new Set());
+	const [pageOffset, setPageOffset] = useMountedState(0);
 
 	const {
 		data: logsSchema,
@@ -140,61 +144,122 @@ const LogTable: FC = () => {
 		setPinnedColumns(new Set());
 		setColumnToggles(new Map());
 	};
-
 	const onRetry = () => {
 		const query = subLogQuery.get();
-		const data = subGapTime.get();
+		resetStreamData();
+		resetLogsData();
+		resetColumns();
+		setPageOffset(0);
 
-		if (logsSchema) {
-			resetStreamData();
-			resetLogsData();
-		}
-		if (data) {
+		if (query) {
 			getQueryData({
-				streamName: subLogQuery.get().streamName,
-				startTime: data.startTime,
-				endTime: data.endTime,
-				access: subLogQuery.get().access,
+				streamName: query.streamName,
+				startTime: query.startTime,
+				endTime: query.endTime,
+				limit: loadLimit,
+				pageOffset: 0,
 			});
+			getDataSchema(query.streamName);
 		}
-
-		getDataSchema(query.streamName);
-		setColumnToggles(new Map());
 	};
+	// const onRetry = () => {
+	// 	const query = subLogQuery.get();
+	// 	const data = subGapTime.get();
+
+	// 	if (logsSchema) {
+	// 		resetStreamData();
+	// 		resetLogsData();
+	// 		setPageOffset(0);
+	// 	}
+	// 	if (data) {
+	// 		getQueryData({
+	// 			streamName: subLogQuery.get().streamName,
+	// 			startTime: data.startTime,
+	// 			endTime: data.endTime,
+	// 			access: subLogQuery.get().access,
+	// 		});
+	// 	}
+
+	// 	getDataSchema(query.streamName);
+	// 	setColumnToggles(new Map());
+	// };
 
 	useEffect(() => {
-		const streamErrorListener = subLogStreamError.subscribe(setLogStreamError);
-		const logSearchListener = subLogSearch.subscribe(setQuerySearch);
-		const refreshIntervalListener = subRefreshInterval.subscribe(setRefreshInterval);
-		const subID = subGapTime.subscribe((data) => {
-			if (data) {
+		if (subLogQuery.get()) {
+			const query = subLogQuery.get();
+			resetColumns();
+			setPageOffset(0);
+			resetStreamData();
+			resetLogsData();
+			if (query) {
 				getQueryData({
-					streamName: subLogQuery.get().streamName,
-					startTime: data.startTime,
-					endTime: data.endTime,
-					access: subLogQuery.get().access,
+					streamName: query.streamName,
+					startTime: query.startTime,
+					endTime: query.endTime,
+					limit: loadLimit,
+					pageOffset: 0,
 				});
-				getDataSchema(subLogQuery.get().streamName);
-				setColumnToggles(new Map());
-				setPinnedColumns(new Set());
+				getDataSchema(query.streamName);
 			}
-		});
+		}
+	}, []);
 
-		const subLogQueryListener = subLogQuery.subscribe(() => {
+	useEffect(() => {
+		const streamErrorListener = subLogStreamError.subscribe(setLogStreamError);
+		const logSearchListener = subLogSearch.subscribe(setQuerySearch);
+		const refreshIntervalListener = subRefreshInterval.subscribe(setRefreshInterval);
+		// const subID = subGapTime.subscribe((data) => {
+		// 	if (data) {
+		// 		getQueryData({
+		// 			streamName: subLogQuery.get().streamName,
+		// 			startTime: data.startTime,
+		// 			endTime: data.endTime,
+		// 			access: subLogQuery.get().access,
+		// 		});
+		// 		getDataSchema(subLogQuery.get().streamName);
+		// 		setColumnToggles(new Map());
+		// 		setPinnedColumns(new Set());
+		// 	}
+		// });
+
+		const subLogQueryListener = subLogQuery.subscribe((state) => {
+			setPageOffset(0);
 			resetLogsData();
 			resetStreamData();
 			resetColumns();
+
+			if (state) {
+				getQueryData({
+					streamName: state.streamName,
+					startTime: state.startTime,
+					endTime: state.endTime,
+					limit: loadLimit,
+					pageOffset: 0,
+				});
+				getDataSchema(state.streamName);
+			}
 		});
 
 		return () => {
 			streamErrorListener();
 			subLogQueryListener();
-			subID();
+			// subID();
 			refreshIntervalListener();
 			logSearchListener();
 		};
 	}, [logsSchema]);
 
+	useEffect(() => {
+		const state = subLogQuery.get();
+		getQueryData({
+			streamName: state.streamName,
+			startTime: state.startTime,
+			endTime: state.endTime,
+			limit: loadLimit,
+			pageOffset: pageOffset,
+		});
+	}, [pageOffset]);
+
 	useEffect(() => {
 		if (subRefreshInterval.get()) {
 			const interval = setInterval(() => {
@@ -277,6 +342,7 @@ const LogTable: FC = () => {
 	const rightRef = useRef<HTMLDivElement>(null);
 	const pinnedContianerRef = useRef<HTMLDivElement>(null);
 	const [pinnedColumnsWidth, setPinnedColumnsWidth] = useMountedState(0);
+	const pagination = usePagination({ total: pageLogData?.totalPages ?? 1, initialPage: 1 });
 
 	useEffect(() => {
 		if (
@@ -298,10 +364,6 @@ const LogTable: FC = () => {
 
 	return (
 		<Box className={container}>
-			<Box
-				sx={{
-					borderBottom: '1px solid #D4D4D4',
-				}}></Box>
 			<FilterPills />
 			{!(logStreamError || logStreamSchemaError || logsError) ? (
 				Boolean(logsSchema?.fields.length) && Boolean(pageLogData?.data.length) ? (
@@ -390,12 +452,58 @@ const LogTable: FC = () => {
 			<Box className={footerContainer}>
 				<Box></Box>
 				{!loading && !logsLoading ? (
-					<Pagination
+					// <Pagination
+					// 	total={pageLogData?.totalPages || 1}
+					// 	value={pageLogData?.page || 1}
+					// 	onChange={(page) => {
+					// 		goToPage(page, pageLogData?.limit || 1);
+					// 	}}></Pagination>
+
+					<Pagination.Root
 						total={pageLogData?.totalPages || 1}
 						value={pageLogData?.page || 1}
 						onChange={(page) => {
 							goToPage(page, pageLogData?.limit || 1);
-						}}></Pagination>
+							pagination.setPage(page);
+						}}>
+						<Group spacing={5} position="center">
+							<Pagination.First
+								onClick={() => {
+									if (pageOffset !== 0) setPageOffset((value) => value - loadLimit);
+								}}
+								disabled={pageOffset === 0}
+							/>
+							<Pagination.Previous />
+							{pagination.range.map((page) => {
+								if (page === 'dots') {
+									return <Pagination.Dots key={page} />;
+								} else {
+									return (
+										<Pagination.Control
+											value={page}
+											key={page}
+											active={pageLogData?.page === page}
+											onClick={() => {
+												goToPage(page);
+												pagination.setPage(page);
+											}}>
+											{pageLogData?.limit ? page + pageOffset / pageLogData?.limit ?? 1 : page}
+										</Pagination.Control>
+									);
+								}
+							})}
+
+							<Pagination.Next />
+							<Pagination.Last
+								onClick={() => {
+									setPageOffset((value) => {
+										return value + loadLimit;
+									});
+								}}
+								disabled={false}
+							/>
+						</Group>
+					</Pagination.Root>
 				) : (
 					<Loader variant="dots" />
 				)}

diff --git a/src/pages/Logs/index.tsx b/src/pages/Logs/index.tsx
@@ -4,7 +4,7 @@ import { FC } from 'react';
 import LogTable from './LogTable';
 import { useLogsStyles } from './styles';
 import ViewLog from './ViewLog';
-import HeaderPagination from './HeaderPagination';
+// import HeaderPagination from './HeaderPagination';
 
 const Logs: FC = () => {
 	useDocumentTitle('Parseable | Logs');
@@ -14,7 +14,7 @@ const Logs: FC = () => {
 
 	return (
 		<Box className={container}>
-			<HeaderPagination />
+			{/* <HeaderPagination /> */}
 			<LogTable />
 			<ViewLog />
 		</Box>

diff --git a/src/pages/Logs/styles.tsx b/src/pages/Logs/styles.tsx
@@ -24,7 +24,7 @@ export const useLogTableStyles = createStyles((theme) => {
 		container: {
 			position: 'relative',
 			flex: 1,
-			maxHeight: `calc(${heights.screen} - ${HEADER_HEIGHT * 2 + 105}px )`,
+			maxHeight: `calc(${heights.screen} - ${HEADER_HEIGHT * 2 }px )`,
 			display: 'flex',
 			flexDirection: 'column',
 			overflow: 'hidden',
@@ -33,7 +33,7 @@ export const useLogTableStyles = createStyles((theme) => {
 		innerContainer: {
 			position: 'relative',
 			flex: 1,
-			maxHeight: `calc(${heights.screen} - ${HEADER_HEIGHT * 2 + 105}px -200px)`,
+			maxHeight: `calc(${heights.screen} - ${HEADER_HEIGHT * 2 }px)`,
 			display: 'flex',
 			flexDirection: 'column',
 			overflow: 'hidden',