initial commit

.github/workflows/release.yml

@@ -0,0 +1,43 @@
+name: Publish Package to npmjs
+on:
+  release:
+    types: [published]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "16.x"
+          registry-url: "https://registry.npmjs.org"
+      - uses: pnpm/action-setup@v2
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          version: 8
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache@v3
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build
+        run: pnpm build
+      - name: Publish to npmjs
+        run: pnpm publish
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -0,0 +1,4 @@
+node_modules
+**/*.tsbuildinfo
+**/**/.next
+**/**/node_modules

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Matt Rickard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,126 @@
+# @react-llm/headless
+
+Easy-to-use headless React Hooks to run LLMs in the browser with WebGPU. As simple as `useLLM()`.
+
+### [**Live Demo**](https://chat.matt-rickard.com)
+
+![image](assets/demo.webp)
+
+**Features**:
+
+* Supports [Vicuna 13B](https://lmsys.org/blog/2023-03-30-vicuna/)
+* Use custom system prompts and "user:"/"assistant:" role names
+* Completion options like `max tokens` and `stop sequences`
+* No data leaves the browser. Accelerated via WebGPU.
+* Hooks built to 'Bring your own UI'
+* Persistent storage for conversations in browser storage. Hooks for loading and saving conversations.
+* Model caching for faster subsequent loads
+
+## Installation
+
+```bash
+npm install @react-llm/headless
+```
+
+
+## **useLLM** API
+### Types
+```typescript
+// Model Initialization
+init: () => void;
+
+// Model Generation
+send: (msg: string, maxTokens: number, stopSequences: string[]) => void;
+onMessage: (msg: GenerateTextResponse) => void;
+setOnMessage: (cb: (msg: GenerateTextResponse) => void) => void;
+
+// Model Status
+loadingStatus: InitProgressReport;
+isGenerating: boolean;
+gpuDevice: GPUDeviceInfo;
+
+// Model Configuration
+userRoleName: string;
+setUserRoleName: (roleName: string) => void;
+assistantRoleName: string;
+setAssistantRoleName: (roleName: string) => void;
+
+// Conversation Management
+conversation: Conversation | undefined;
+allConversations: Conversation[] | undefined;
+createConversation: (title?: string, prompt?: string) => void;
+setConversationId: (conversationId: string) => void;
+deleteConversation: (conversationId: string) => void;
+deleteAllConversations: () => void;
+deleteMessages: () => void;
+setConversationTitle: (conversationId: string, title: string) => void;
+```
+
+### Hooks
+```typescript
+import useLLM from '@react-llm/headless';
+
+const MyComponent = () => {
+  const {
+    conversation,
+    allConversations,
+    loadingStatus,
+    isGenerating,
+    createConversation,
+    setConversationId,
+    deleteConversation,
+    deleteAllConversations,
+    deleteMessages,
+    setConversationTitle,
+    onMessage,
+    setOnMessage,
+    userRoleName,
+    setUserRoleName,
+    assistantRoleName,
+    setAssistantRoleName,
+    gpuDevice,
+    send,
+    init,
+  } = useLLM();
+
+  // Component logic...
+
+  return null;
+};
+```
+
+
+### Packages
+
+* `@react-llm/headless` - Headless React Hooks for running LLMs in the browser
+* `@react-llm/retro-ui` - Retro-themed UI for the hooks
+
+## How does it work?
+
+This library is a set of React Hooks that provide a simple interface to run LLMs in the browser. It uses Vicuna 13B.
+
+* SentencePiece tokenizer (compiled for the browser via Emscripten)
+* Vicuna 13B (transformed to Apache TVM format)
+* Apache TVM and MLC Relax (compiled for the browser via Emscripten)
+* Off-the-main-thread WebWorker to run the model (bundled with the library)
+
+
+The model, tokenizer, and TVM runtime are loaded from a CDN (huggingface). The model is cached in browser storage for faster subsequent loads.
+
+
+
+
+### Example
+See [packages/retro-ui](packages/retro-ui) for the full demo code. This is a simple example of how to use the hooks. To run it, after cloning the repo,
+
+```bash
+cd packages/retro-ui
+pnpm install
+pnpm dev
+```
+
+
+### License
+MIT
+
+The code under `packages/headless/worker/lib/tvm` is licensed under Apache 2.0.

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@react-llm/workspace",
+  "version": "0.0.1",
+  "type": "module",
+  "main": "dist/bundle.cjs.js",
+  "module": "dist/bundle.esm.js",
+  "author": "Matt Rickard <[email protected]>",
+  "license": "MIT",
+  "private": true,
+  "workspaces": [
+    "packages/headless",
+    "packages/retro-ui"
+  ],
+  "scripts": {
+    "publish": "pnpm publish --access public",
+    "build": "pnpm recursive run build"
+  },
+  "devDependencies": {
+    "typescript": "^5.0.4"
+  },
+  "dependencies": {
+    "react95": "^4.0.0",
+    "styled-components": "^5.3.10"
+  }
+}

packages/headless/.eslintrc.json

@@ -0,0 +1,5 @@
+{
+  "extends": [
+    "@typescript-eslint/no-unused-vars"
+  ]
+}

packages/headless/.gitignore

@@ -0,0 +1,35 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

packages/headless/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Matt Rickard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.