feat: talk with URL

package.json

@@ -55,6 +55,7 @@
     "react": "^18.2.0",
     "redent": "^4.0.0",
     "tiktoken": "^1.0.14",
+    "turndown": "^7.1.3",
     "update-notifier": "^7.0.0",
     "yargs": "^17.7.2",
     "zod": "^3.23.6",
@@ -67,6 +68,7 @@
     "@types/mock-fs": "^4.13.4",
     "@types/prompts": "^2.4.9",
     "@types/react": "^18.2.63",
+    "@types/turndown": "^5.0.4",
     "@types/update-notifier": "^6.0.8",
     "del-cli": "^5.1.0",
     "eslint": "^8.57.0",

src/commands/chat/index.tsx

@@ -16,10 +16,10 @@ export const command: CommandModule<{}, PromptOptions> = {
   handler: (args) => run(args._.join(' '), args),
 };
 
-function run(initialPrompt: string, options: PromptOptions) {
+async function run(initialPrompt: string, options: PromptOptions) {
   try {
     const configFile = parseConfigFile();
-    initChatState(options, configFile, initialPrompt);
+    await initChatState(options, configFile, initialPrompt);
     render(
       <ThemeProvider theme={inkTheme}>
         <ChatUi />

src/commands/chat/prompt-options.ts

@@ -8,6 +8,8 @@ export interface PromptOptions {
   model?: string;
   /** Add file to conversation */
   file?: string;
+  /** Add website to the conversation */
+  url?: string;
   /** Creative response style */
   creative?: boolean;
   /** Precise response style */
@@ -34,6 +36,10 @@ export const promptOptions: Record<keyof PromptOptions, Options> = {
     type: 'string',
     describe: 'Add given file to conversation context',
   },
+  'url': {
+    type: 'string',
+    describe: 'Add given website to conversation context',
+  },
   'creative': {
     type: 'boolean',
     describe: 'Enable more creative responses',

src/commands/chat/state/init.ts

@@ -1,13 +1,14 @@
+import TurndownService from 'turndown';
 import { type ConfigFile } from '../../../config-file.js';
-import { DEFAULT_SYSTEM_PROMPT } from '../../../default-config.js';
+import { DEFAULT_FILE_PROMPT, DEFAULT_SYSTEM_PROMPT } from '../../../default-config.js';
 import type { ResponseStyle } from '../../../engine/providers/config.js';
 import type { Message } from '../../../engine/inference.js';
 import type { PromptOptions } from '../prompt-options.js';
 import { getDefaultProvider, resolveProviderFromOption } from '../providers.js';
 import { filterOutApiKey, handleInputFile } from '../utils.js';
 import { useChatState, type ChatMessage, type ChatState } from './state.js';
 
-export function initChatState(
+export async function initChatState(
   options: PromptOptions,
   configFile: ConfigFile,
   initialPrompt: string,
@@ -67,6 +68,37 @@ export function initChatState(
     }
   }
 
+  if (options.url) {
+    const urlResponse = await fetch(options.url);
+    const urlContent = await urlResponse.text();
+    const turndown = new TurndownService();
+    turndown.addRule('removeScript', {
+      filter: 'script',
+      replacement: function () {
+        return '';
+      },
+    });
+    turndown.addRule('removeStyle', {
+      filter: 'style',
+      replacement: function () {
+        return '';
+      },
+    });
+    turndown.addRule('removeImage', {
+      filter: 'img',
+      replacement: function () {
+        return '';
+      },
+    });
+    const urlMarkdown = turndown.turndown(urlContent);
+
+    const content = DEFAULT_FILE_PROMPT.replace('{filename}', options.url).replace(
+      '{fileContent}',
+      urlMarkdown,
+    );
+    contextMessages.push({ role: 'system', content });
+  }
+
   if (initialPrompt) {
     contextMessages.push({ role: 'user', content: initialPrompt });
     outputMessages.push({ type: 'user', text: initialPrompt });

yarn.lock

@@ -703,6 +703,7 @@ __metadata:
     "@types/mock-fs": "npm:^4.13.4"
     "@types/prompts": "npm:^2.4.9"
     "@types/react": "npm:^18.2.63"
+    "@types/turndown": "npm:^5.0.4"
     "@types/update-notifier": "npm:^6.0.8"
     chalk: "npm:^5.3.0"
     date-fns: "npm:^3.6.0"
@@ -725,6 +726,7 @@ __metadata:
     redent: "npm:^4.0.0"
     release-it: "npm:^15.11.0"
     tiktoken: "npm:^1.0.14"
+    turndown: "npm:^7.1.3"
     typescript: "npm:^5.4.5"
     update-notifier: "npm:^7.0.0"
     yargs: "npm:^17.7.2"
@@ -1713,6 +1715,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/turndown@npm:^5.0.4":
+  version: 5.0.4
+  resolution: "@types/turndown@npm:5.0.4"
+  checksum: 10/500ba3a40952a5e41d99e407d83df70066408eb78d92e5f1f5f27bf70939aad11dcb64ae36872289b9c11948aa9bc4242d63d4898ad4bd91e9a41e1eb368c24f
+  languageName: node
+  linkType: hard
+
 "@types/update-notifier@npm:^6.0.8":
   version: 6.0.8
   resolution: "@types/update-notifier@npm:6.0.8"
@@ -3521,6 +3530,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"domino@npm:^2.1.6":
+  version: 2.1.6
+  resolution: "domino@npm:2.1.6"
+  checksum: 10/cc2fb95084dd7cdf5df073191768807d70c8bef0e08e92cf5ce02d02933d6e6ecc4c742ac4985e5adfb93466776980280b9c6fb93fb4fd6b4ec85368c9f5e053
+  languageName: node
+  linkType: hard
+
 "dot-prop@npm:^5.1.0":
   version: 5.3.0
   resolution: "dot-prop@npm:5.3.0"
@@ -9213,6 +9229,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"turndown@npm:^7.1.3":
+  version: 7.1.3
+  resolution: "turndown@npm:7.1.3"
+  dependencies:
+    domino: "npm:^2.1.6"
+  checksum: 10/a3622c4a26b9079dfad4e4b004fc8c3a6a7f786ae0ba1ccaa78d1de71094594977413fccce62dc68dc45d521cf1864d6a253c886403122f7824e9244179dc740
+  languageName: node
+  linkType: hard
+
 "type-check@npm:^0.4.0, type-check@npm:~0.4.0":
   version: 0.4.0
   resolution: "type-check@npm:0.4.0"