use infinite loader in collection memories

elpassion · Jan 23, 2025 · 61268c9 · 61268c9
1 parent 09d753c
commit 61268c9
Show file tree

Hide file tree

Showing 9 changed files with 77 additions and 21 deletions.
diff --git a/apps/web-remix/app/api/knowledgeBase/KnowledgeBaseApi.ts b/apps/web-remix/app/api/knowledgeBase/KnowledgeBaseApi.ts
@@ -67,11 +67,14 @@ export class KnowledgeBaseApi {
   async getCollectionMemories(
     organizationId: string | number,
     collectionId: string | number,
+    queryParams?: Partial<PaginationQueryParams>,
   ) {
-    return this.client(
-      KnowledgeBaseFileListResponse,
+    const url = buildUrlWithParams(
       `/organizations/${organizationId}/memory_collections/${collectionId}/memories`,
+      { ...queryParams },
     );
+
+    return this.client(KnowledgeBaseFileListResponse, url);
   }
 
   async getCollectionCosts(

diff --git a/apps/web-remix/app/api/knowledgeBase/knowledgeApi.contracts.ts b/apps/web-remix/app/api/knowledgeBase/knowledgeApi.contracts.ts
@@ -124,8 +124,9 @@ export const KnowledgeBaseCollectionList = z.array(KnowledgeBaseCollection);
 export const KnowledgeBaseFileListResponse = z
   .object({
     data: KnowledgeBaseFileList,
+    meta: PaginationMeta,
   })
-  .transform((res) => res.data);
+  .transform((res) => ({ data: res.data, meta: res.meta }));
 
 export type IKnowledgeBaseFileListResponse = z.TypeOf<
   typeof KnowledgeBaseFileListResponse

diff --git a/apps/web-remix/app/components/pages/knowledgeBase/collectionContent/loader.server.tsx b/apps/web-remix/app/components/pages/knowledgeBase/collectionContent/loader.server.tsx
@@ -3,6 +3,7 @@ import type { LoaderFunctionArgs } from '@remix-run/node';
 import invariant from 'tiny-invariant';
 
 import { KnowledgeBaseApi } from '~/api/knowledgeBase/KnowledgeBaseApi';
+import { getParamsPagination } from '~/components/pagination/usePagination';
 import { requireLogin } from '~/session.server';
 import { loaderBuilder } from '~/utils.server';
 
@@ -13,6 +14,11 @@ export async function loader(args: LoaderFunctionArgs) {
     invariant(params.collectionName, 'collectionName not found');
 
     const knowledgeBaseApi = new KnowledgeBaseApi(fetch);
+    const searchParams = new URL(request.url).searchParams;
+
+    const { page, per_page, ...rest } = getParamsPagination(searchParams, {
+      per_page: 40,
+    });
 
     const {
       data: { id: collectionId },
@@ -24,12 +30,17 @@ export async function loader(args: LoaderFunctionArgs) {
     const knowledgeBase = await knowledgeBaseApi.getCollectionMemories(
       params.organizationId,
       collectionId,
+      { page, per_page },
     );
 
+    const totalItems = knowledgeBase.data.meta.total;
+    const totalPages = Math.ceil(totalItems / per_page);
+
     return json({
-      fileList: knowledgeBase.data,
+      fileList: knowledgeBase.data.data,
       organizationId: params.organizationId,
       collectionName: params.collectionName,
+      pagination: { page, per_page, totalPages, totalItems, ...rest },
     });
   })(args);
 }
diff --git a/apps/web-remix/app/components/pages/knowledgeBase/collectionContent/page.tsx b/apps/web-remix/app/components/pages/knowledgeBase/collectionContent/page.tsx
@@ -1,4 +1,5 @@
-import React, { useMemo } from 'react';
+import React, { useEffect, useMemo } from 'react';
+import { useInView } from 'react-intersection-observer';
 import type { MetaFunction } from '@remix-run/node';
 import {
   Outlet,
@@ -16,7 +17,12 @@ import {
   ListActionProvider,
   useListAction,
 } from '~/components/pages/knowledgeBase/components/ListActionProvider';
-import type { IKnowledgeBaseFileList } from '~/components/pages/knowledgeBase/knowledgeBase.types';
+import type {
+  IKnowledgeBaseFile,
+  IKnowledgeBaseFileList,
+} from '~/components/pages/knowledgeBase/knowledgeBase.types';
+import { LoadMoreButton } from '~/components/pagination/LoadMoreButton';
+import { useInfiniteFetch } from '~/components/pagination/useInfiniteFetch';
 import { Button } from '~/components/ui/button';
 import {
   DialogDrawer,
@@ -35,10 +41,25 @@ import { KnowledgeBaseFileList } from './KnowledgeBaseFileList';
 import type { loader } from './loader.server';
 
 export function KnowledgeBaseContentPage() {
-  const { fileList, organizationId, collectionName } =
+  const { ref: fetchNextRef, inView } = useInView();
+  const navigate = useNavigate();
+
+  const { fileList, organizationId, collectionName, pagination } =
     useLoaderData<typeof loader>();
 
-  const navigate = useNavigate();
+  const { hasNextPage, data, fetchNextPage, isFetchingNextPage } =
+    useInfiniteFetch<IKnowledgeBaseFile, typeof loader>({
+      pagination,
+      initialData: fileList,
+      loaderUrl: routes.collectionFiles(organizationId, collectionName),
+      dataExtractor: (response) => response.data?.fileList,
+    });
+
+  useEffect(() => {
+    if (inView && !isFetchingNextPage && hasNextPage) {
+      fetchNextPage();
+    }
+  }, [inView, isFetchingNextPage, hasNextPage]);
 
   const matchNew = useMatch(
     `:organizationId/knowledge-base/:collectionName/content/new`,
@@ -73,7 +94,7 @@ export function KnowledgeBaseContentPage() {
     <ListActionProvider>
       <PageContentWrapper className="mt-5">
         <div className="flex justify-between gap-2 items-center  mb-4">
-          {fileList.length > 0 ? <SelectAllButton items={fileList} /> : null}
+          {data.length > 0 ? <SelectAllButton items={data} /> : null}
 
           <Button asChild className="w-fit ml-auto mr-0 flex">
             <BasicLink
@@ -84,7 +105,16 @@ export function KnowledgeBaseContentPage() {
           </Button>
         </div>
 
-        <KnowledgeBaseFileList items={fileList} />
+        <KnowledgeBaseFileList items={data} />
+
+        <div className="flex justify-center mt-5" ref={fetchNextRef}>
+          <LoadMoreButton
+            isFetching={isFetchingNextPage}
+            hasNextPage={hasNextPage}
+            onClick={fetchNextPage}
+            className="text-xs"
+          />
+        </div>
 
         <DialogDrawer open={isSidebarOpen} onOpenChange={handleClose}>
           <DialogDrawerContent

diff --git a/apps/web-remix/app/components/pages/knowledgeBase/collectionGraph/loader.server.tsx b/apps/web-remix/app/components/pages/knowledgeBase/collectionGraph/loader.server.tsx
@@ -24,7 +24,9 @@ export async function loader(args: LoaderFunctionArgs) {
     const extend_neighbors =
       url.searchParams.get('extend_neighbors') === 'true';
     const extend_parents = url.searchParams.get('extend_parents') === 'true';
-    const memory_id = url.searchParams.get('memory_id') ? Number(url.searchParams.get('memory_id')) : undefined;
+    const memory_id = url.searchParams.get('memory_id')
+      ? Number(url.searchParams.get('memory_id'))
+      : undefined;
 
     const knowledgeBaseApi = new KnowledgeBaseApi(fetch);
 
@@ -92,7 +94,7 @@ export async function loader(args: LoaderFunctionArgs) {
       prevNode,
       nextNode,
       searchParams,
-      fileList: knowledgeBase.data,
+      fileList: knowledgeBase.data.data,
     });
   })(args);
 }
diff --git a/apps/web-remix/app/components/pages/knowledgeBase/knowledgeBaseSearch/loader.server.tsx b/apps/web-remix/app/components/pages/knowledgeBase/knowledgeBaseSearch/loader.server.tsx
@@ -72,7 +72,7 @@ export async function loader(args: LoaderFunctionArgs) {
       metadata,
       organizationId: params.organizationId,
       collectionName: params.collectionName,
-      fileList: knowledgeBase.data,
+      fileList: knowledgeBase.data.data,
     });
   })(args);
 }
diff --git a/apps/web-remix/app/components/pages/knowledgeBase/knowledgeBaseSearch/page.tsx b/apps/web-remix/app/components/pages/knowledgeBase/knowledgeBaseSearch/page.tsx
@@ -8,15 +8,19 @@ import { KnowledgeBaseSearchList } from './KnowledgeBaseSearchList';
 import type { loader } from './loader.server';
 
 export function KnowledgeBaseSearch() {
-  const { chunks, metadata, queryMetadata, fileList } = useLoaderData<typeof loader>();
+  const { chunks, metadata, queryMetadata, fileList } =
+    useLoaderData<typeof loader>();
 
   return (
     <div className="p-1">
       <p className="text-sm text-muted-foreground mb-2">
         Total tokens: {metadata.total_tokens}
       </p>
 
-      <KnowledgeBaseSearchForm defaultValue={queryMetadata} fileList={fileList} />
+      <KnowledgeBaseSearchForm
+        defaultValue={queryMetadata}
+        fileList={fileList}
+      />
 
       <div className="mt-4">
         <KnowledgeBaseSearchList items={chunks} query={queryMetadata.query} />

diff --git a/apps/web-remix/app/components/pages/pipelines/list/loader.server.tsx b/apps/web-remix/app/components/pages/pipelines/list/loader.server.tsx
@@ -21,8 +21,8 @@ export async function loader(args: LoaderFunctionArgs) {
     if (!searchParams.has('page')) {
       searchParams.set('per_page', '30');
     }
-
     const { page, per_page, search, sort } = getParamsPagination(searchParams);
+
     const pipelinesPromise = pipelineApi.getPipelines(params.organizationId, {
       page,
       per_page,

diff --git a/apps/web-remix/app/components/pagination/usePagination.tsx b/apps/web-remix/app/components/pagination/usePagination.tsx
@@ -63,11 +63,16 @@ export const usePagination = (initialPagination?: Partial<Pagination>) => {
   };
 };
 
-export function getParamsPagination(params: URLSearchParams) {
+export function getParamsPagination(
+  params: URLSearchParams,
+  defaults?: Partial<typeof DEFAULT_PAGINATION>,
+) {
+  const defaultPagination = { ...DEFAULT_PAGINATION, ...defaults };
+
   return {
-    page: Number(params.get('page')) || DEFAULT_PAGINATION.page,
-    per_page: Number(params.get('per_page')) || DEFAULT_PAGINATION.per_page,
-    search: params.get('search') || DEFAULT_PAGINATION.search,
-    sort: params.get('sort') || DEFAULT_PAGINATION.sort,
+    page: Number(params.get('page')) || defaultPagination.page,
+    per_page: Number(params.get('per_page')) || defaultPagination.per_page,
+    search: params.get('search') || defaultPagination.search,
+    sort: params.get('sort') || defaultPagination.sort,
   };
 }