Skip to content

Commit

Permalink
fix: embeddings (#169)
Browse files Browse the repository at this point in the history
* fix: embeddings

* fix: formatting
  • Loading branch information
royscheepens authored Nov 15, 2024
1 parent a28bc41 commit 9a3bdd6
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 13 deletions.
File renamed without changes.
25 changes: 16 additions & 9 deletions lib/generate-embeddings.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { createHash } from 'node:crypto';
import { readFile, readdir, stat } from 'node:fs/promises';
import { basename, dirname, join } from 'node:path';
import { createClient } from '@supabase/supabase-js';
import dotenv from 'dotenv';
import { readFile, readdir, stat } from 'node:fs/promises';
import GithubSlugger from 'github-slugger';
import { Content, Root } from 'mdast';
import { fromMarkdown } from 'mdast-util-from-markdown';
Expand Down Expand Up @@ -66,11 +66,12 @@ function extractMetaTags(mdxTree: Root) {
* @param slug
* @returns
*/

const parseMetaTitle = (meta: any, slug: string): string => {
if (!meta[slug]) return slug;

if (typeof meta[slug] === 'object') {
return `${(meta[slug] as any).title}` ?? slug;
return meta[slug]?.title ? `${(meta[slug] as any).title}` : slug;
}

return meta[slug] as string;
Expand Down Expand Up @@ -236,15 +237,20 @@ class MarkdownEmbeddingSource extends BaseEmbeddingSource {
async load() {
const contents = await readFile(this.filePath, 'utf8');

const slug = this.filePath
.split('/')
.at(-1)
.replace(/\.mdx?$/, '');
const slug =
this.filePath
.split('/')
.at(-1)
?.replace(/\.mdx?$/, '') ?? '';

const metaPath = join(
process.cwd(),
this.filePath.replace(/[^/]+$/, '_meta.ts'),
);

const metaPath = this.filePath.replace(/[^/]+$/, '_meta.json');
const metaJson = await readFile(metaPath, 'utf8');
const metaFile = (await import(metaPath)).default;

const title = parseMetaTitle(JSON.parse(metaJson), slug);
const title = parseMetaTitle(metaFile, slug);

const { checksum, meta, sections } = processMdxForSearch(title, contents);

Expand All @@ -263,6 +269,7 @@ class MarkdownEmbeddingSource extends BaseEmbeddingSource {
type EmbeddingSource = MarkdownEmbeddingSource;

async function generateEmbeddings() {
// @ts-ignore
const argv = await yargs.option('refresh', {
alias: 'r',
description: 'Refresh data',
Expand Down
7 changes: 3 additions & 4 deletions pages/_app.tsx
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import React from 'react';
import { Analytics } from '@vercel/analytics/react';
import { CookieConsentModal } from '@onbeam/features';

import { Analytics } from '@vercel/analytics/react';
import React from 'react';
import { GoogleTagManager } from '../components/google-tag-manager';
import '../styles.css';
import { GoogleTagManager } from '../lib/GoogleTagManager';

export default function Nextra({ Component, pageProps }) {
return (
Expand Down

0 comments on commit 9a3bdd6

Please sign in to comment.