From dfe042558e186118039d562bca1af70ced937081 Mon Sep 17 00:00:00 2001 From: Maciej Jastrzebski Date: Wed, 8 May 2024 14:44:45 +0200 Subject: [PATCH] wip --- package.json | 2 ++ src/commands/chat/index.tsx | 4 ++-- src/commands/chat/prompt-options.ts | 6 +++++ src/commands/chat/state/init.ts | 36 +++++++++++++++++++++++++++-- yarn.lock | 25 ++++++++++++++++++++ 5 files changed, 69 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 8ff033a..1f76208 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "react": "^18.2.0", "redent": "^4.0.0", "tiktoken": "^1.0.14", + "turndown": "^7.1.3", "update-notifier": "^7.0.0", "yargs": "^17.7.2", "zod": "^3.23.6", @@ -67,6 +68,7 @@ "@types/mock-fs": "^4.13.4", "@types/prompts": "^2.4.9", "@types/react": "^18.2.63", + "@types/turndown": "^5.0.4", "@types/update-notifier": "^6.0.8", "del-cli": "^5.1.0", "eslint": "^8.57.0", diff --git a/src/commands/chat/index.tsx b/src/commands/chat/index.tsx index b48d393..4ab16f4 100644 --- a/src/commands/chat/index.tsx +++ b/src/commands/chat/index.tsx @@ -16,10 +16,10 @@ export const command: CommandModule<{}, PromptOptions> = { handler: (args) => run(args._.join(' '), args), }; -function run(initialPrompt: string, options: PromptOptions) { +async function run(initialPrompt: string, options: PromptOptions) { try { const configFile = parseConfigFile(); - initChatState(options, configFile, initialPrompt); + await initChatState(options, configFile, initialPrompt); render( diff --git a/src/commands/chat/prompt-options.ts b/src/commands/chat/prompt-options.ts index 52518a5..9ed601b 100644 --- a/src/commands/chat/prompt-options.ts +++ b/src/commands/chat/prompt-options.ts @@ -8,6 +8,8 @@ export interface PromptOptions { model?: string; /** Add file to conversation */ file?: string; + /** Add website to the conversation */ + url?: string; /** Creative response style */ creative?: boolean; /** Precise response style */ @@ -34,6 +36,10 @@ export const promptOptions: Record = { type: 'string', describe: 'Add given file to conversation context', }, + 'url': { + type: 'string', + describe: 'Add given website to conversation context', + }, 'creative': { type: 'boolean', describe: 'Enable more creative responses', diff --git a/src/commands/chat/state/init.ts b/src/commands/chat/state/init.ts index c793838..7ee5986 100644 --- a/src/commands/chat/state/init.ts +++ b/src/commands/chat/state/init.ts @@ -1,5 +1,6 @@ +import TurndownService from 'turndown'; import { type ConfigFile } from '../../../config-file.js'; -import { DEFAULT_SYSTEM_PROMPT } from '../../../default-config.js'; +import { DEFAULT_FILE_PROMPT, DEFAULT_SYSTEM_PROMPT } from '../../../default-config.js'; import type { ResponseStyle } from '../../../engine/providers/config.js'; import type { Message } from '../../../engine/inference.js'; import type { PromptOptions } from '../prompt-options.js'; @@ -7,7 +8,7 @@ import { getDefaultProvider, resolveProviderFromOption } from '../providers.js'; import { filterOutApiKey, handleInputFile } from '../utils.js'; import { useChatState, type ChatMessage, type ChatState } from './state.js'; -export function initChatState( +export async function initChatState( options: PromptOptions, configFile: ConfigFile, initialPrompt: string, @@ -67,6 +68,37 @@ export function initChatState( } } + if (options.url) { + const urlResponse = await fetch(options.url); + const urlContent = await urlResponse.text(); + const turndown = new TurndownService(); + turndown.addRule('removeScript', { + filter: 'script', + replacement: function () { + return ''; + }, + }); + turndown.addRule('removeStyle', { + filter: 'style', + replacement: function () { + return ''; + }, + }); + turndown.addRule('removeImage', { + filter: 'img', + replacement: function () { + return ''; + }, + }); + const urlMarkdown = turndown.turndown(urlContent); + + const content = DEFAULT_FILE_PROMPT.replace('{filename}', options.url).replace( + '{fileContent}', + urlMarkdown, + ); + contextMessages.push({ role: 'system', content }); + } + if (initialPrompt) { contextMessages.push({ role: 'user', content: initialPrompt }); outputMessages.push({ type: 'user', text: initialPrompt }); diff --git a/yarn.lock b/yarn.lock index f16d0f8..1be59b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -703,6 +703,7 @@ __metadata: "@types/mock-fs": "npm:^4.13.4" "@types/prompts": "npm:^2.4.9" "@types/react": "npm:^18.2.63" + "@types/turndown": "npm:^5.0.4" "@types/update-notifier": "npm:^6.0.8" chalk: "npm:^5.3.0" date-fns: "npm:^3.6.0" @@ -725,6 +726,7 @@ __metadata: redent: "npm:^4.0.0" release-it: "npm:^15.11.0" tiktoken: "npm:^1.0.14" + turndown: "npm:^7.1.3" typescript: "npm:^5.4.5" update-notifier: "npm:^7.0.0" yargs: "npm:^17.7.2" @@ -1713,6 +1715,13 @@ __metadata: languageName: node linkType: hard +"@types/turndown@npm:^5.0.4": + version: 5.0.4 + resolution: "@types/turndown@npm:5.0.4" + checksum: 10/500ba3a40952a5e41d99e407d83df70066408eb78d92e5f1f5f27bf70939aad11dcb64ae36872289b9c11948aa9bc4242d63d4898ad4bd91e9a41e1eb368c24f + languageName: node + linkType: hard + "@types/update-notifier@npm:^6.0.8": version: 6.0.8 resolution: "@types/update-notifier@npm:6.0.8" @@ -3521,6 +3530,13 @@ __metadata: languageName: node linkType: hard +"domino@npm:^2.1.6": + version: 2.1.6 + resolution: "domino@npm:2.1.6" + checksum: 10/cc2fb95084dd7cdf5df073191768807d70c8bef0e08e92cf5ce02d02933d6e6ecc4c742ac4985e5adfb93466776980280b9c6fb93fb4fd6b4ec85368c9f5e053 + languageName: node + linkType: hard + "dot-prop@npm:^5.1.0": version: 5.3.0 resolution: "dot-prop@npm:5.3.0" @@ -9213,6 +9229,15 @@ __metadata: languageName: node linkType: hard +"turndown@npm:^7.1.3": + version: 7.1.3 + resolution: "turndown@npm:7.1.3" + dependencies: + domino: "npm:^2.1.6" + checksum: 10/a3622c4a26b9079dfad4e4b004fc8c3a6a7f786ae0ba1ccaa78d1de71094594977413fccce62dc68dc45d521cf1864d6a253c886403122f7824e9244179dc740 + languageName: node + linkType: hard + "type-check@npm:^0.4.0, type-check@npm:~0.4.0": version: 0.4.0 resolution: "type-check@npm:0.4.0"