Merge remote-tracking branch 'origin/dev'

.github/workflows/client-release.yml

@@ -1,32 +1,32 @@
-
-name: Go Build to release
+name: goreleaser
 
 on:
   release:
     types: [released]
   workflow_dispatch:
 
+permissions:
+  contents: write
+
 jobs:
-  releases-matrix:
-    name: Release Go Binary
+  goreleaser:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # build and publish in parallel: linux/386, linux/amd64, linux/arm64, windows/386, windows/amd64, darwin/amd64, darwin/arm64
-        goos: [linux, windows, darwin]
-        goarch: ["386", amd64, arm64]
-        exclude:
-          - goarch: "386"
-            goos: darwin
-          - goarch: arm64
-            goos: windows
-      fail-fast: false
     steps:
-    - uses: actions/checkout@v3
-    - uses: wangyoucao577/go-release-action@v1
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        goos: ${{ matrix.goos }}
-        goarch: ${{ matrix.goarch }}
-        goversion: "https://dl.google.com/go/go1.20.2.linux-amd64.tar.gz"
-        binary_name: "go-ai-cli"
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v5
+        with:
+          distribution: goreleaser
+          version: latest
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          # Uncomment and set GORELEASER_KEY if using GoReleaser Pro
+          # GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

.github/workflows/test.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v2
       with:
-        go-version: '^1.20.4'
+        go-version: '^1.22'
 
     - name: Build
       run: go build ./...

.vscode/launch.json

@@ -4,15 +4,24 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Launch test function",
+      "type": "go",
+      "request": "launch",
+      "mode": "test",
+      "program": "${file}",
+      "args": [],
+      "showLog": true
+    },
     {
       "name": "Launch file",
       "type": "go",
       "request": "launch",
       "mode": "debug",
       "program": "main.go",
       "args": ["prompt"],
-      "console": "integratedTerminal"
+      "console": "integratedTerminal",
+      "buildFlags": ["-tags", "portaudio"],
     }
-
   ]
-}
+}

api/agent/agent_test.go

@@ -0,0 +1,39 @@
+package agent_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/MohammadBnei/go-ai-cli/api"
+	"github.com/MohammadBnei/go-ai-cli/api/agent"
+	"github.com/MohammadBnei/go-ai-cli/config"
+	"github.com/spf13/viper"
+	"github.com/tmc/langchaingo/chains"
+	"github.com/tmc/langchaingo/llms/openai"
+)
+
+func TestWebSearchAgent(t *testing.T) {
+	viper.Set(config.AI_API_TYPE, api.API_OLLAMA)
+	viper.Set(config.AI_MODEL_NAME, "based-dolphin-mistral")
+	viper.Set(config.AI_OLLAMA_HOST, "http://127.0.0.1:11434")
+
+	t.Log("TestWebSearchAgent")
+	llm, err := openai.New(openai.WithModel("gpt-4-turbo-preview"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("Created llm")
+
+	executor, err := agent.NewAutoWebSearchAgent(llm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log("Created executor")
+
+	result, err := chains.Run(context.Background(), executor, "I want you to design a system prompt for a gpt-4 model. The system prompt will make the ai model tell persian mythology. I want the model to respond with true facts, not to invent anything. The model will respond in the format of an engaging story, presenting itself as a character inside the story.")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Log("Result: " + result)
+}

api/agent/autoWebSearch.go

@@ -0,0 +1,77 @@
+package agent
+
+import (
+	"context"
+
+	"github.com/MohammadBnei/go-ai-cli/api"
+	"github.com/tmc/langchaingo/agents"
+	"github.com/tmc/langchaingo/chains"
+	"github.com/tmc/langchaingo/llms"
+	"github.com/tmc/langchaingo/prompts"
+	"github.com/tmc/langchaingo/tools"
+	"github.com/tmc/langchaingo/tools/duckduckgo"
+	"github.com/tmc/langchaingo/tools/scraper"
+)
+
+func NewAutoWebSearchAgent(llm llms.Model) (*agents.Executor, error) {
+	ddg, err := duckduckgo.New(10, "en")
+	if err != nil {
+		return nil, err
+	}
+
+	scrap, err := scraper.New()
+	if err != nil {
+		return nil, err
+	}
+
+	t := []tools.Tool{
+		NewSearchInputDesigner(),
+		ddg,
+		scrap,
+	}
+
+	executor, err := agents.Initialize(llm, t, agents.ZeroShotReactDescription,
+		agents.WithMaxIterations(5),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &executor, nil
+}
+
+type SearchInputDesigner struct {
+}
+
+func NewSearchInputDesigner() tools.Tool {
+	return &SearchInputDesigner{}
+
+}
+
+func (s *SearchInputDesigner) Name() string {
+	return "SearchInputDesigner"
+}
+
+func (s *SearchInputDesigner) Description() string {
+	return "SearchInputDesigner is a tool designed to help users design search inputs."
+}
+
+func (s *SearchInputDesigner) Call(ctx context.Context, input string) (string, error) {
+	llm, err := api.GetLlmModel()
+	if err != nil {
+		return "", err
+	}
+
+	prompt := prompts.NewPromptTemplate(
+		"Design a search input, to be used by search engines, for the following text: {{.userInput}}?",
+		[]string{"userInput"},
+	)
+
+	chain := chains.NewLLMChain(llm, prompt)
+	out, err := chains.Run(ctx, chain, input)
+	if err != nil {
+		return "", err
+	}
+
+	return out, nil
+}

api/agent/webSearch.go

@@ -0,0 +1,123 @@
+package agent
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/MohammadBnei/go-ai-cli/api"
+	"github.com/chromedp/chromedp"
+	"github.com/gocolly/colly/v2"
+	"github.com/tmc/langchaingo/chains"
+	"github.com/tmc/langchaingo/llms"
+	"github.com/tmc/langchaingo/prompts"
+	"github.com/tmc/langchaingo/schema"
+	"github.com/tmc/langchaingo/tools/scraper"
+	"golang.org/x/sync/errgroup"
+)
+
+func NewWebSearchAgent(llm llms.Model, urls []string) (func(ctx context.Context, question string) (map[string]any, error), error) {
+	var g errgroup.Group
+
+	if len(urls) > 3 {
+		return nil, errors.New("too many urls (3 max for now)")
+	}
+
+	docs := make([]schema.Document, len(urls))
+	for i, url := range urls {
+		g.Go(func(url string, i int) func() error {
+			return func() error {
+				data, err := fetchHTML(url)
+				if err != nil {
+					return err
+				}
+
+				parsedData, err := parseHtml(context.Background(), data)
+				if err != nil {
+					return err
+				}
+
+				docs[i] = schema.Document{
+					PageContent: parsedData,
+					Metadata: map[string]any{
+						"url": url,
+					},
+				}
+
+				return nil
+			}
+		}(url, i))
+	}
+
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+
+	stuffQAChain := chains.LoadStuffQA(llm)
+
+	callFunc := func(ctx context.Context, question string) (map[string]any, error) {
+		return chains.Call(context.Background(), stuffQAChain, map[string]any{
+			"input_documents": docs,
+			"question":        question,
+		})
+	}
+
+	return callFunc, nil
+}
+
+func getHtmlContent(url string) (string, error) {
+	c := colly.NewCollector(
+		colly.UserAgent("Mozilla/5.0 (Windows NT  10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"),
+	)
+	c.WithTransport(&http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	})
+
+	scrap, err := scraper.New()
+	if err != nil {
+		return "", err
+	}
+	return scrap.Call(context.Background(), url)
+}
+
+func fetchHTML(url string) (string, error) {
+	// Initialize a new browser context
+	ctx, cancel := chromedp.NewContext(context.Background())
+	defer cancel()
+
+	// Navigate to the URL and fetch the rendered HTML
+	var htmlContent string
+	err := chromedp.Run(ctx,
+		chromedp.Navigate(url),
+		chromedp.OuterHTML("html", &htmlContent),
+	)
+	if err != nil {
+		return "", err
+	}
+
+	return htmlContent, nil
+}
+func parseHtml(ctx context.Context, data string) (string, error) {
+
+	llm, err := api.GetLlmModel()
+	if err != nil {
+		return "", err
+	}
+
+	prompt := prompts.NewPromptTemplate(
+		`You are an HTML parser. You job is to extract meaningful information from the HTML, removing the html code while keeping the meaning. 
+		If you see a XSS security, a "is human" verification, it means that the site is not accessible to you. If you see something like this, this is the output you must produce : site not accessible : [SHOW ERROR HERE].		Here is the content : {{.html}}`,
+		[]string{"html"},
+	)
+
+	chain := chains.NewLLMChain(llm, prompt)
+	out, err := chains.Run(ctx, chain, data)
+	if err != nil {
+	}
+
+	fmt.Println("data :"+data, "\nout :"+out)
+
+	return out, nil
+}