From 9aeb1cd77754ec77cfe43fe0ddfbfb5939e20eb1 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 4 Nov 2024 22:43:14 -0700 Subject: [PATCH] chore: make cred stores be tools --- pkg/cli/credential.go | 27 +--- pkg/cli/credential_delete.go | 19 +-- pkg/cli/credential_show.go | 19 +-- pkg/config/cliconfig.go | 72 +++------- pkg/credentials/factory.go | 77 ++++++++++ pkg/credentials/noop.go | 4 +- pkg/credentials/runnerprogram.go | 29 ++++ pkg/credentials/store.go | 82 ++++------- pkg/credentials/{helper.go => toolstore.go} | 18 +-- pkg/credentials/util.go | 49 ------- pkg/engine/cmd.go | 3 + pkg/engine/engine.go | 3 - pkg/gptscript/gptscript.go | 85 ++++++++--- pkg/parser/parser.go | 6 + pkg/repos/get.go | 149 +------------------- pkg/repos/runtimes/golang/golang.go | 13 -- pkg/runner/runner.go | 5 - pkg/runner/runtimemanager.go | 9 -- pkg/sdkserver/credentials.go | 17 +-- pkg/types/tool.go | 4 + 20 files changed, 268 insertions(+), 422 deletions(-) create mode 100644 pkg/credentials/factory.go create mode 100644 pkg/credentials/runnerprogram.go rename pkg/credentials/{helper.go => toolstore.go} (82%) delete mode 100644 pkg/credentials/util.go diff --git a/pkg/cli/credential.go b/pkg/cli/credential.go index a46c483b..eaf7665b 100644 --- a/pkg/cli/credential.go +++ b/pkg/cli/credential.go @@ -9,10 +9,7 @@ import ( "time" cmd2 "github.com/gptscript-ai/cmd" - "github.com/gptscript-ai/gptscript/pkg/config" - "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/gptscript" - "github.com/gptscript-ai/gptscript/pkg/repos/runtimes" "github.com/spf13/cobra" ) @@ -37,33 +34,19 @@ func (c *Credential) Customize(cmd *cobra.Command) { } func (c *Credential) Run(cmd *cobra.Command, _ []string) error { - cfg, err := config.ReadCLIConfig(c.root.ConfigFile) - if err != nil { - return fmt.Errorf("failed to read CLI config: %w", err) - } - opts, err := c.root.NewGPTScriptOpts() if err != nil { return err } - opts = gptscript.Complete(opts) - if opts.Runner.RuntimeManager == nil { - opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir) - } - - ctxs := opts.CredentialContexts - if c.AllContexts { - ctxs = []string{credentials.AllCredentialContexts} - } - - if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { + gptScript, err := gptscript.New(cmd.Context(), opts) + if err != nil { return err } + defer gptScript.Close(true) - // Initialize the credential store and get all the credentials. - store, err := credentials.NewStore(cfg, opts.Runner.RuntimeManager, ctxs, opts.Cache.CacheDir) + store, err := gptScript.CredentialStoreFactory.NewStore(gptScript.DefaultCredentialContexts) if err != nil { - return fmt.Errorf("failed to get credentials store: %w", err) + return err } creds, err := store.List(cmd.Context()) diff --git a/pkg/cli/credential_delete.go b/pkg/cli/credential_delete.go index 81392f36..6c43a41b 100644 --- a/pkg/cli/credential_delete.go +++ b/pkg/cli/credential_delete.go @@ -3,10 +3,7 @@ package cli import ( "fmt" - "github.com/gptscript-ai/gptscript/pkg/config" - "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/gptscript" - "github.com/gptscript-ai/gptscript/pkg/repos/runtimes" "github.com/spf13/cobra" ) @@ -28,23 +25,15 @@ func (c *Delete) Run(cmd *cobra.Command, args []string) error { return err } - cfg, err := config.ReadCLIConfig(c.root.ConfigFile) + gptScript, err := gptscript.New(cmd.Context(), opts) if err != nil { - return fmt.Errorf("failed to read CLI config: %w", err) - } - - opts = gptscript.Complete(opts) - if opts.Runner.RuntimeManager == nil { - opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir) - } - - if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { return err } + defer gptScript.Close(true) - store, err := credentials.NewStore(cfg, opts.Runner.RuntimeManager, opts.CredentialContexts, opts.Cache.CacheDir) + store, err := gptScript.CredentialStoreFactory.NewStore(gptScript.DefaultCredentialContexts) if err != nil { - return fmt.Errorf("failed to get credentials store: %w", err) + return err } if err = store.Remove(cmd.Context(), args[0]); err != nil { diff --git a/pkg/cli/credential_show.go b/pkg/cli/credential_show.go index ab2e9cd1..95cb4f11 100644 --- a/pkg/cli/credential_show.go +++ b/pkg/cli/credential_show.go @@ -5,10 +5,7 @@ import ( "os" "text/tabwriter" - "github.com/gptscript-ai/gptscript/pkg/config" - "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/gptscript" - "github.com/gptscript-ai/gptscript/pkg/repos/runtimes" "github.com/spf13/cobra" ) @@ -30,23 +27,15 @@ func (c *Show) Run(cmd *cobra.Command, args []string) error { return err } - cfg, err := config.ReadCLIConfig(c.root.ConfigFile) + gptScript, err := gptscript.New(cmd.Context(), opts) if err != nil { - return fmt.Errorf("failed to read CLI config: %w", err) - } - - opts = gptscript.Complete(opts) - if opts.Runner.RuntimeManager == nil { - opts.Runner.RuntimeManager = runtimes.Default(opts.Cache.CacheDir, opts.SystemToolsDir) - } - - if err = opts.Runner.RuntimeManager.SetUpCredentialHelpers(cmd.Context(), cfg); err != nil { return err } + defer gptScript.Close(true) - store, err := credentials.NewStore(cfg, opts.Runner.RuntimeManager, opts.CredentialContexts, opts.Cache.CacheDir) + store, err := gptScript.CredentialStoreFactory.NewStore(gptScript.DefaultCredentialContexts) if err != nil { - return fmt.Errorf("failed to get credentials store: %w", err) + return err } cred, exists, err := store.Get(cmd.Context(), args[0]) diff --git a/pkg/config/cliconfig.go b/pkg/config/cliconfig.go index 215e2f39..73741ab3 100644 --- a/pkg/config/cliconfig.go +++ b/pkg/config/cliconfig.go @@ -3,11 +3,9 @@ package config import ( "encoding/base64" "encoding/json" - "errors" "fmt" "os" "runtime" - "slices" "strings" "sync" @@ -21,28 +19,13 @@ const ( SecretserviceCredHelper = "secretservice" PassCredHelper = "pass" FileCredHelper = "file" - SqliteCredHelper = "sqlite" - PostgresCredHelper = "postgres" - - GPTScriptHelperPrefix = "gptscript-credential-" ) var ( - darwinHelpers = []string{OsxkeychainCredHelper, FileCredHelper, SqliteCredHelper, PostgresCredHelper} - windowsHelpers = []string{WincredCredHelper, FileCredHelper} - linuxHelpers = []string{SecretserviceCredHelper, PassCredHelper, FileCredHelper, SqliteCredHelper, PostgresCredHelper} + // Helpers is a list of all supported credential helpers from github.com/gptscript-ai/gptscript-credential-helpers + Helpers = []string{WincredCredHelper, OsxkeychainCredHelper, SecretserviceCredHelper, PassCredHelper} ) -func listAsString(helpers []string) string { - if len(helpers) == 0 { - return "" - } else if len(helpers) == 1 { - return helpers[0] - } - - return strings.Join(helpers[:len(helpers)-1], ", ") + " or " + helpers[len(helpers)-1] -} - type AuthConfig types.AuthConfig func (a AuthConfig) MarshalJSON() ([]byte, error) { @@ -74,8 +57,8 @@ func (a *AuthConfig) UnmarshalJSON(data []byte) error { type CLIConfig struct { Auths map[string]AuthConfig `json:"auths,omitempty"` CredentialsStore string `json:"credsStore,omitempty"` - Integrations map[string]string `json:"integrations,omitempty"` + raw []byte auths map[string]types.AuthConfig authsLock *sync.Mutex location string @@ -108,7 +91,19 @@ func (c *CLIConfig) Save() error { } c.auths = nil } - data, err := json.Marshal(c) + + // This is to not overwrite additional fields that might be the config file + out := map[string]any{} + if len(c.raw) > 0 { + err := json.Unmarshal(c.raw, &out) + if err != nil { + return err + } + } + out["auths"] = c.Auths + out["credsStore"] = c.CredentialsStore + + data, err := json.Marshal(out) if err != nil { return err } @@ -154,34 +149,22 @@ func ReadCLIConfig(gptscriptConfigFile string) (*CLIConfig, error) { result := &CLIConfig{ authsLock: &sync.Mutex{}, location: gptscriptConfigFile, + raw: data, } if err := json.Unmarshal(data, result); err != nil { return nil, fmt.Errorf("failed to unmarshal %s: %v", gptscriptConfigFile, err) } + if store := os.Getenv("GPTSCRIPT_CREDENTIAL_STORE"); store != "" { + result.CredentialsStore = store + } + if result.CredentialsStore == "" { if err := result.setDefaultCredentialsStore(); err != nil { return nil, err } } - if !isValidCredentialHelper(result.CredentialsStore) { - errMsg := fmt.Sprintf("invalid credential store '%s'", result.CredentialsStore) - switch runtime.GOOS { - case "darwin": - errMsg += fmt.Sprintf(" (use %s)", listAsString(darwinHelpers)) - case "windows": - errMsg += fmt.Sprintf(" (use %s)", listAsString(windowsHelpers)) - case "linux": - errMsg += fmt.Sprintf(" (use %s)", listAsString(linuxHelpers)) - default: - errMsg += " (use file)" - } - errMsg += fmt.Sprintf("\nPlease edit your config file at %s to fix this.", result.location) - - return nil, errors.New(errMsg) - } - return result, nil } @@ -197,19 +180,6 @@ func (c *CLIConfig) setDefaultCredentialsStore() error { return c.Save() } -func isValidCredentialHelper(helper string) bool { - switch runtime.GOOS { - case "darwin": - return slices.Contains(darwinHelpers, helper) - case "windows": - return slices.Contains(windowsHelpers, helper) - case "linux": - return slices.Contains(linuxHelpers, helper) - default: - return helper == FileCredHelper - } -} - func readFile(path string) ([]byte, error) { data, err := os.ReadFile(path) if os.IsNotExist(err) { diff --git a/pkg/credentials/factory.go b/pkg/credentials/factory.go new file mode 100644 index 00000000..ca6f1d18 --- /dev/null +++ b/pkg/credentials/factory.go @@ -0,0 +1,77 @@ +package credentials + +import ( + "context" + + "github.com/docker/docker-credential-helpers/client" + "github.com/gptscript-ai/gptscript/pkg/config" + "github.com/gptscript-ai/gptscript/pkg/types" +) + +type ProgramLoaderRunner interface { + Load(ctx context.Context, toolName string) (prg types.Program, err error) + Run(ctx context.Context, prg types.Program, input string) (output string, err error) +} + +func NewFactory(ctx context.Context, cfg *config.CLIConfig, plr ProgramLoaderRunner) (StoreFactory, error) { + toolName := translateToolName(cfg.CredentialsStore) + if toolName == config.FileCredHelper { + return StoreFactory{ + file: true, + cfg: cfg, + }, nil + } + + prg, err := plr.Load(ctx, toolName) + if err != nil { + return StoreFactory{}, err + } + + return StoreFactory{ + ctx: ctx, + prg: prg, + runner: plr, + cfg: cfg, + }, nil +} + +type StoreFactory struct { + ctx context.Context + prg types.Program + file bool + runner ProgramLoaderRunner + cfg *config.CLIConfig +} + +func (s *StoreFactory) NewStore(credCtxs []string) (CredentialStore, error) { + if err := validateCredentialCtx(credCtxs); err != nil { + return nil, err + } + if s.file { + return Store{ + credCtxs: credCtxs, + cfg: s.cfg, + }, nil + } + return Store{ + credCtxs: credCtxs, + cfg: s.cfg, + program: s.program, + }, nil +} + +func (s *StoreFactory) program(args ...string) client.Program { + return &runnerProgram{ + factory: s, + action: args[0], + } +} + +func translateToolName(toolName string) string { + for _, helper := range config.Helpers { + if helper == toolName { + return "github.com/gptscript-ai/gptscript-credential-helpers/" + toolName + "/cmd" + } + } + return toolName +} diff --git a/pkg/credentials/noop.go b/pkg/credentials/noop.go index 3a13b907..414f8a12 100644 --- a/pkg/credentials/noop.go +++ b/pkg/credentials/noop.go @@ -1,6 +1,8 @@ package credentials -import "context" +import ( + "context" +) type NoopStore struct{} diff --git a/pkg/credentials/runnerprogram.go b/pkg/credentials/runnerprogram.go new file mode 100644 index 00000000..4ae123a0 --- /dev/null +++ b/pkg/credentials/runnerprogram.go @@ -0,0 +1,29 @@ +package credentials + +import ( + "io" +) + +type runnerProgram struct { + factory *StoreFactory + action string + output string + err error +} + +func (r *runnerProgram) Output() ([]byte, error) { + return []byte(r.output), r.err +} + +func (r *runnerProgram) Input(in io.Reader) { + input, err := io.ReadAll(in) + if err != nil { + r.err = err + return + } + + prg := r.factory.prg + prg.EntryToolID = prg.ToolSet[prg.EntryToolID].LocalTools[r.action] + + r.output, r.err = r.factory.runner.Run(r.factory.ctx, prg, string(input)) +} diff --git a/pkg/credentials/store.go b/pkg/credentials/store.go index 1843cd8d..56555cd4 100644 --- a/pkg/credentials/store.go +++ b/pkg/credentials/store.go @@ -3,13 +3,12 @@ package credentials import ( "context" "fmt" - "path/filepath" "regexp" "slices" - "strings" "github.com/docker/cli/cli/config/credentials" "github.com/docker/cli/cli/config/types" + "github.com/docker/docker-credential-helpers/client" credentials2 "github.com/docker/docker-credential-helpers/credentials" "github.com/gptscript-ai/gptscript/pkg/config" "golang.org/x/exp/maps" @@ -20,10 +19,6 @@ const ( AllCredentialContexts = "*" ) -type CredentialBuilder interface { - EnsureCredentialHelpers(ctx context.Context) error -} - type CredentialStore interface { Get(ctx context.Context, toolName string) (*Credential, bool, error) Add(ctx context.Context, cred Credential) error @@ -33,30 +28,17 @@ type CredentialStore interface { } type Store struct { - credCtxs []string - credBuilder CredentialBuilder - credHelperDirs CredentialHelperDirs - cfg *config.CLIConfig + credCtxs []string + cfg *config.CLIConfig + program client.ProgramFunc } -func NewStore(cfg *config.CLIConfig, credentialBuilder CredentialBuilder, credCtxs []string, cacheDir string) (CredentialStore, error) { - if err := validateCredentialCtx(credCtxs); err != nil { - return nil, err - } - return Store{ - credCtxs: credCtxs, - credBuilder: credentialBuilder, - credHelperDirs: GetCredentialHelperDirs(cacheDir, cfg.CredentialsStore), - cfg: cfg, - }, nil -} - -func (s Store) Get(ctx context.Context, toolName string) (*Credential, bool, error) { - if first(s.credCtxs) == AllCredentialContexts { +func (s Store) Get(_ context.Context, toolName string) (*Credential, bool, error) { + if len(s.credCtxs) > 0 && s.credCtxs[0] == AllCredentialContexts { return nil, false, fmt.Errorf("cannot get a credential with context %q", AllCredentialContexts) } - store, err := s.getStore(ctx) + store, err := s.getStore() if err != nil { return nil, false, err } @@ -99,14 +81,14 @@ func (s Store) Get(ctx context.Context, toolName string) (*Credential, bool, err // Add adds a new credential to the credential store. // Any context set on the credential object will be overwritten with the first context of the credential store. -func (s Store) Add(ctx context.Context, cred Credential) error { +func (s Store) Add(_ context.Context, cred Credential) error { first := first(s.credCtxs) if first == AllCredentialContexts { return fmt.Errorf("cannot add a credential with context %q", AllCredentialContexts) } cred.Context = first - store, err := s.getStore(ctx) + store, err := s.getStore() if err != nil { return err } @@ -118,12 +100,12 @@ func (s Store) Add(ctx context.Context, cred Credential) error { } // Refresh updates an existing credential in the credential store. -func (s Store) Refresh(ctx context.Context, cred Credential) error { +func (s Store) Refresh(_ context.Context, cred Credential) error { if !slices.Contains(s.credCtxs, cred.Context) { return fmt.Errorf("context %q not in list of valid contexts for this credential store", cred.Context) } - store, err := s.getStore(ctx) + store, err := s.getStore() if err != nil { return err } @@ -134,13 +116,13 @@ func (s Store) Refresh(ctx context.Context, cred Credential) error { return store.Store(auth) } -func (s Store) Remove(ctx context.Context, toolName string) error { +func (s Store) Remove(_ context.Context, toolName string) error { first := first(s.credCtxs) if len(s.credCtxs) > 1 || first == AllCredentialContexts { return fmt.Errorf("error: credential deletion is not supported when multiple credential contexts are provided") } - store, err := s.getStore(ctx) + store, err := s.getStore() if err != nil { return err } @@ -148,8 +130,8 @@ func (s Store) Remove(ctx context.Context, toolName string) error { return store.Erase(toolNameWithCtx(toolName, first)) } -func (s Store) List(ctx context.Context) ([]Credential, error) { - store, err := s.getStore(ctx) +func (s Store) List(_ context.Context) ([]Credential, error) { + store, err := s.getStore() if err != nil { return nil, err } @@ -179,7 +161,7 @@ func (s Store) List(ctx context.Context) ([]Credential, error) { } } - if first(s.credCtxs) == AllCredentialContexts { + if len(s.credCtxs) > 0 && s.credCtxs[0] == AllCredentialContexts { return allCreds, nil } @@ -194,25 +176,14 @@ func (s Store) List(ctx context.Context) ([]Credential, error) { return maps.Values(credsByName), nil } -func (s *Store) getStore(ctx context.Context) (credentials.Store, error) { - return s.getStoreByHelper(ctx, config.GPTScriptHelperPrefix+s.cfg.CredentialsStore) -} - -func (s *Store) getStoreByHelper(ctx context.Context, helper string) (credentials.Store, error) { - if helper == "" || helper == config.GPTScriptHelperPrefix+config.FileCredHelper { - return credentials.NewFileStore(s.cfg), nil +func (s *Store) getStore() (credentials.Store, error) { + if s.program != nil { + return &toolCredentialStore{ + file: credentials.NewFileStore(s.cfg), + program: s.program, + }, nil } - - // If the helper is referencing one of the credential helper programs, then reference the full path. - if strings.HasPrefix(helper, "gptscript-credential-") { - if err := s.credBuilder.EnsureCredentialHelpers(ctx); err != nil { - return nil, err - } - - helper = filepath.Join(s.credHelperDirs.BinDir, helper) - } - - return NewHelper(s.cfg, helper) + return credentials.NewFileStore(s.cfg), nil } func validateCredentialCtx(ctxs []string) error { @@ -234,3 +205,10 @@ func validateCredentialCtx(ctxs []string) error { return nil } + +func first(s []string) string { + if len(s) == 0 { + return "" + } + return s[0] +} diff --git a/pkg/credentials/helper.go b/pkg/credentials/toolstore.go similarity index 82% rename from pkg/credentials/helper.go rename to pkg/credentials/toolstore.go index e5cd34f6..3e31ea12 100644 --- a/pkg/credentials/helper.go +++ b/pkg/credentials/toolstore.go @@ -10,22 +10,14 @@ import ( "github.com/docker/cli/cli/config/types" "github.com/docker/docker-credential-helpers/client" credentials2 "github.com/docker/docker-credential-helpers/credentials" - "github.com/gptscript-ai/gptscript/pkg/config" ) -func NewHelper(c *config.CLIConfig, helper string) (credentials.Store, error) { - return &HelperStore{ - file: credentials.NewFileStore(c), - program: client.NewShellProgramFunc(helper), - }, nil -} - -type HelperStore struct { +type toolCredentialStore struct { file credentials.Store program client.ProgramFunc } -func (h *HelperStore) Erase(serverAddress string) error { +func (h *toolCredentialStore) Erase(serverAddress string) error { var errs []error if err := client.Erase(h.program, serverAddress); err != nil { errs = append(errs, err) @@ -36,7 +28,7 @@ func (h *HelperStore) Erase(serverAddress string) error { return errors.Join(errs...) } -func (h *HelperStore) Get(serverAddress string) (types.AuthConfig, error) { +func (h *toolCredentialStore) Get(serverAddress string) (types.AuthConfig, error) { creds, err := client.Get(h.program, serverAddress) if credentials2.IsErrCredentialsNotFound(err) { return h.file.Get(serverAddress) @@ -50,7 +42,7 @@ func (h *HelperStore) Get(serverAddress string) (types.AuthConfig, error) { }, nil } -func (h *HelperStore) GetAll() (map[string]types.AuthConfig, error) { +func (h *toolCredentialStore) GetAll() (map[string]types.AuthConfig, error) { result := map[string]types.AuthConfig{} serverAddresses, err := client.List(h.program) @@ -103,7 +95,7 @@ func (h *HelperStore) GetAll() (map[string]types.AuthConfig, error) { return result, nil } -func (h *HelperStore) Store(authConfig types.AuthConfig) error { +func (h *toolCredentialStore) Store(authConfig types.AuthConfig) error { return client.Store(h.program, &credentials2.Credentials{ ServerURL: authConfig.ServerAddress, Username: authConfig.Username, diff --git a/pkg/credentials/util.go b/pkg/credentials/util.go deleted file mode 100644 index 2a4ad96b..00000000 --- a/pkg/credentials/util.go +++ /dev/null @@ -1,49 +0,0 @@ -package credentials - -import ( - "fmt" - "path/filepath" - - "github.com/gptscript-ai/gptscript/pkg/config" - runtimeEnv "github.com/gptscript-ai/gptscript/pkg/env" -) - -type CredentialHelperDirs struct { - RevisionFile, LastCheckedFile, BinDir string -} - -func RepoNameForCredentialStore(store string) string { - switch store { - case config.SqliteCredHelper, config.PostgresCredHelper: - return "gptscript-credential-database" - default: - return "gptscript-credential-helpers" - } -} - -func GitURLForRepoName(repoName string) (string, error) { - switch repoName { - case "gptscript-credential-database": - return runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_SQLITE_ROOT", "https://github.com/gptscript-ai/gptscript-credential-database.git"), nil - case "gptscript-credential-helpers": - return runtimeEnv.VarOrDefault("GPTSCRIPT_CRED_HELPERS_ROOT", "https://github.com/gptscript-ai/gptscript-credential-helpers.git"), nil - default: - return "", fmt.Errorf("unknown repo name: %s", repoName) - } -} - -func GetCredentialHelperDirs(cacheDir, store string) CredentialHelperDirs { - repoName := RepoNameForCredentialStore(store) - return CredentialHelperDirs{ - RevisionFile: filepath.Join(cacheDir, "repos", repoName, "revision"), - LastCheckedFile: filepath.Join(cacheDir, "repos", repoName, "last-checked"), - BinDir: filepath.Join(cacheDir, "repos", repoName, "bin"), - } -} - -func first(s []string) string { - if len(s) == 0 { - return "" - } - return s[0] -} diff --git a/pkg/engine/cmd.go b/pkg/engine/cmd.go index a4f6d3ed..b0c1ab4b 100644 --- a/pkg/engine/cmd.go +++ b/pkg/engine/cmd.go @@ -149,6 +149,9 @@ func (e *Engine) runCommand(ctx Context, tool types.Tool, input string, toolCate result *bytes.Buffer ) + if tool.Stdin { + cmd.Stdin = strings.NewReader(input) + } cmd.Stdout = io.MultiWriter(stdout, stdoutAndErr, progressOut) cmd.Stderr = io.MultiWriter(stdoutAndErr, progressOut, os.Stderr) result = stdout diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 44ed50bb..7232157e 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -7,7 +7,6 @@ import ( "strings" "sync" - "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/counter" "github.com/gptscript-ai/gptscript/pkg/types" "github.com/gptscript-ai/gptscript/pkg/version" @@ -20,8 +19,6 @@ type Model interface { type RuntimeManager interface { GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) - EnsureCredentialHelpers(ctx context.Context) error - SetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig) error } type Engine struct { diff --git a/pkg/gptscript/gptscript.go b/pkg/gptscript/gptscript.go index df4b0792..63a67647 100644 --- a/pkg/gptscript/gptscript.go +++ b/pkg/gptscript/gptscript.go @@ -2,6 +2,7 @@ package gptscript import ( "context" + "errors" "fmt" "os" "os/user" @@ -16,6 +17,7 @@ import ( "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/llm" + "github.com/gptscript-ai/gptscript/pkg/loader" "github.com/gptscript-ai/gptscript/pkg/monitor" "github.com/gptscript-ai/gptscript/pkg/mvl" "github.com/gptscript-ai/gptscript/pkg/openai" @@ -32,13 +34,15 @@ import ( var log = mvl.Package() type GPTScript struct { - Registry *llm.Registry - Runner *runner.Runner - Cache *cache.Client - WorkspacePath string - DeleteWorkspaceOnClose bool - ExtraEnv []string - close func() + Registry *llm.Registry + Runner *runner.Runner + Cache *cache.Client + CredentialStoreFactory credentials.StoreFactory + DefaultCredentialContexts []string + WorkspacePath string + DeleteWorkspaceOnClose bool + ExtraEnv []string + close func() } type Options struct { @@ -103,11 +107,17 @@ func New(ctx context.Context, o ...Options) (*GPTScript, error) { opts.Runner.RuntimeManager = runtimes.Default(cacheClient.CacheDir(), opts.SystemToolsDir) } - if err := opts.Runner.RuntimeManager.SetUpCredentialHelpers(context.Background(), cliCfg); err != nil { + simplerRunner, err := newSimpleRunner(cacheClient, opts.Runner.RuntimeManager, opts.Env) + if err != nil { + return nil, err + } + + storeFactory, err := credentials.NewFactory(ctx, cliCfg, simplerRunner) + if err != nil { return nil, err } - credStore, err := credentials.NewStore(cliCfg, opts.Runner.RuntimeManager, opts.CredentialContexts, cacheClient.CacheDir()) + credStore, err := storeFactory.NewStore(opts.CredentialContexts) if err != nil { return nil, err } @@ -158,13 +168,15 @@ func New(ctx context.Context, o ...Options) (*GPTScript, error) { } return &GPTScript{ - Registry: registry, - Runner: runner, - Cache: cacheClient, - WorkspacePath: opts.Workspace, - DeleteWorkspaceOnClose: opts.Workspace == "", - ExtraEnv: extraEnv, - close: closeServer, + Registry: registry, + Runner: runner, + Cache: cacheClient, + CredentialStoreFactory: storeFactory, + DefaultCredentialContexts: opts.CredentialContexts, + WorkspacePath: opts.Workspace, + DeleteWorkspaceOnClose: opts.Workspace == "", + ExtraEnv: extraEnv, + close: closeServer, }, nil } @@ -266,3 +278,44 @@ func (g *GPTScript) ListTools(_ context.Context, prg types.Program) []types.Tool func (g *GPTScript) ListModels(ctx context.Context, providers ...string) ([]string, error) { return g.Registry.ListModels(ctx, providers...) } + +type simpleRunner struct { + cache *cache.Client + runner *runner.Runner + env []string +} + +func newSimpleRunner(cache *cache.Client, rm engine.RuntimeManager, env []string) (*simpleRunner, error) { + runner, err := runner.New(noopModel{}, credentials.NoopStore{}, runner.Options{ + RuntimeManager: rm, + }) + if err != nil { + return nil, err + } + return &simpleRunner{ + cache: cache, + runner: runner, + env: env, + }, nil +} + +func (s *simpleRunner) Load(ctx context.Context, toolName string) (prg types.Program, err error) { + return loader.Program(ctx, toolName, "", loader.Options{ + Cache: s.cache, + }) +} + +func (s *simpleRunner) Run(ctx context.Context, prg types.Program, input string) (output string, err error) { + return s.runner.Run(ctx, prg, s.env, input) +} + +type noopModel struct { +} + +func (n noopModel) Call(_ context.Context, _ types.CompletionRequest, _ []string, _ chan<- types.CompletionStatus) (*types.CompletionMessage, error) { + return nil, errors.New("unsupported") +} + +func (n noopModel) ProxyInfo() (string, string, error) { + return "", "", errors.New("unsupported") +} diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index c0beb8f2..626056a7 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -124,6 +124,12 @@ func isParam(line string, tool *types.Tool, scan *simplescanner) (_ bool, err er tool.Parameters.ExportContext = append(tool.Parameters.ExportContext, csv(scan.AddMultiline(value))...) case "context": tool.Parameters.Context = append(tool.Parameters.Context, csv(scan.AddMultiline(value))...) + case "stdin": + b, err := toBool(value) + if err != nil { + return false, err + } + tool.Parameters.Stdin = b case "metadata": mkey, mvalue, _ := strings.Cut(scan.AddMultiline(value), ":") if tool.MetaData == nil { diff --git a/pkg/repos/get.go b/pkg/repos/get.go index 0a50ce15..d77fb526 100644 --- a/pkg/repos/get.go +++ b/pkg/repos/get.go @@ -4,22 +4,15 @@ import ( "context" "encoding/json" "errors" - "fmt" "io/fs" "os" "path/filepath" "regexp" - "runtime" "strings" - "sync" - "time" "github.com/BurntSushi/locker" - "github.com/gptscript-ai/gptscript/pkg/config" - "github.com/gptscript-ai/gptscript/pkg/credentials" "github.com/gptscript-ai/gptscript/pkg/hash" "github.com/gptscript-ai/gptscript/pkg/repos/git" - "github.com/gptscript-ai/gptscript/pkg/repos/runtimes/golang" "github.com/gptscript-ai/gptscript/pkg/types" ) @@ -55,19 +48,12 @@ func (n noopRuntime) Setup(_ context.Context, _ types.Tool, _, _ string, _ []str } type Manager struct { - cacheDir string - storageDir string - gitDir string - runtimeDir string - systemDirs []string - runtimes []Runtime - credHelperConfig *credHelperConfig -} - -type credHelperConfig struct { - lock sync.Mutex - initialized bool - cliCfg *config.CLIConfig + cacheDir string + storageDir string + gitDir string + runtimeDir string + systemDirs []string + runtimes []Runtime } func New(cacheDir, systemDir string, runtimes ...Runtime) *Manager { @@ -90,129 +76,6 @@ func New(cacheDir, systemDir string, runtimes ...Runtime) *Manager { } } -func (m *Manager) EnsureCredentialHelpers(ctx context.Context) error { - if m.credHelperConfig == nil { - return nil - } - m.credHelperConfig.lock.Lock() - defer m.credHelperConfig.lock.Unlock() - - if !m.credHelperConfig.initialized { - if err := m.deferredSetUpCredentialHelpers(ctx, m.credHelperConfig.cliCfg); err != nil { - return err - } - m.credHelperConfig.initialized = true - } - - return nil -} - -func (m *Manager) SetUpCredentialHelpers(_ context.Context, cliCfg *config.CLIConfig) error { - m.credHelperConfig = &credHelperConfig{ - cliCfg: cliCfg, - } - return nil -} - -func (m *Manager) deferredSetUpCredentialHelpers(ctx context.Context, cliCfg *config.CLIConfig) error { - var ( - helperName = cliCfg.CredentialsStore - distInfo, suffix string - ) - // The file helper is built-in and does not need to be downloaded. - if helperName == config.FileCredHelper { - return nil - } - switch helperName { - case config.WincredCredHelper: - suffix = ".exe" - default: - distInfo = fmt.Sprintf("-%s-%s", runtime.GOOS, runtime.GOARCH) - } - - repoName := credentials.RepoNameForCredentialStore(helperName) - - locker.Lock(repoName) - defer locker.Unlock(repoName) - - credHelperDirs := credentials.GetCredentialHelperDirs(m.cacheDir, helperName) - - // Load the last-checked file to make sure we haven't checked the repo in the last 24 hours. - now := time.Now() - lastChecked, err := os.ReadFile(credHelperDirs.LastCheckedFile) - if err == nil { - if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(lastChecked))); err == nil && now.Sub(t) < 24*time.Hour { - // Make sure the binary still exists, and if it does, return. - if _, err := os.Stat(filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { - log.Debugf("Credential helper %s up-to-date as of %v, checking for updates after %v", helperName, t, t.Add(24*time.Hour)) - return nil - } - } - } - - if err := os.MkdirAll(filepath.Dir(credHelperDirs.LastCheckedFile), 0755); err != nil { - return err - } - - // Update the last-checked file. - if err := os.WriteFile(credHelperDirs.LastCheckedFile, []byte(now.Format(time.RFC3339)), 0644); err != nil { - return err - } - - gitURL, err := credentials.GitURLForRepoName(repoName) - if err != nil { - return err - } - - tool := types.Tool{ - ToolDef: types.ToolDef{ - Parameters: types.Parameters{ - Name: repoName, - }, - }, - Source: types.ToolSource{ - Repo: &types.Repo{ - Root: gitURL, - }, - }, - } - tag, err := golang.GetLatestTag(tool) - if err != nil { - return err - } - - var needsDownloaded bool - // Check the last revision shasum and see if it is different from the current one. - lastRevision, err := os.ReadFile(credHelperDirs.RevisionFile) - if (err == nil && strings.TrimSpace(string(lastRevision)) != tool.Source.Repo.Root+tag) || errors.Is(err, fs.ErrNotExist) { - // Need to pull the latest version. - needsDownloaded = true - // Update the revision file to the new revision. - if err = os.WriteFile(credHelperDirs.RevisionFile, []byte(tool.Source.Repo.Root+tag), 0644); err != nil { - return err - } - } else if err != nil { - return err - } - - if !needsDownloaded { - // Check for the existence of the credential helper binary. - // If it's there, we have no need to download it and can just return. - if _, err = os.Stat(filepath.Join(credHelperDirs.BinDir, "gptscript-credential-"+helperName+suffix)); err == nil { - return nil - } - } - - // Find the Go runtime and use it to build the credential helper. - for _, rt := range m.runtimes { - if strings.HasPrefix(rt.ID(), "go") { - return rt.(*golang.Runtime).DownloadCredentialHelper(ctx, tool, helperName, distInfo, suffix, credHelperDirs.BinDir) - } - } - - return fmt.Errorf("no Go runtime found to build the credential helper") -} - func (m *Manager) setup(ctx context.Context, runtime Runtime, tool types.Tool, env []string) (string, []string, error) { locker.Lock(tool.ID) defer locker.Unlock(tool.ID) diff --git a/pkg/repos/runtimes/golang/golang.go b/pkg/repos/runtimes/golang/golang.go index 23c12f1a..5cba4779 100644 --- a/pkg/repos/runtimes/golang/golang.go +++ b/pkg/repos/runtimes/golang/golang.go @@ -100,19 +100,6 @@ type tag struct { } `json:"commit"` } -func GetLatestTag(tool types.Tool) (string, error) { - r, ok, err := getLatestRelease(tool) - if err != nil { - return "", err - } - - if !ok { - return "", fmt.Errorf("failed to get latest release for %s", tool.Name) - } - - return r.label, nil -} - func getLatestRelease(tool types.Tool) (*release, bool, error) { if tool.Source.Repo == nil || !strings.HasPrefix(tool.Source.Repo.Root, "https://github.com/") { return nil, false, nil diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 18bc1bc4..37a90d48 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -862,11 +862,6 @@ func (r *Runner) handleCredentials(callCtx engine.Context, monitor Monitor, env refresh bool ) - rm := runtimeWithLogger(callCtx, monitor, r.runtimeManager) - if err := rm.EnsureCredentialHelpers(callCtx.Ctx); err != nil { - return nil, fmt.Errorf("failed to setup credential helpers: %w", err) - } - // Only try to look up the cred if the tool is on GitHub or has an alias. // If it is a GitHub tool and has an alias, the alias overrides the tool name, so we use it as the credential name. if isGitHubTool(toolName) && credentialAlias == "" { diff --git a/pkg/runner/runtimemanager.go b/pkg/runner/runtimemanager.go index ed191d15..1c293215 100644 --- a/pkg/runner/runtimemanager.go +++ b/pkg/runner/runtimemanager.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/gptscript-ai/gptscript/pkg/config" "github.com/gptscript-ai/gptscript/pkg/engine" "github.com/gptscript-ai/gptscript/pkg/mvl" "github.com/gptscript-ai/gptscript/pkg/types" @@ -40,11 +39,3 @@ func (r runtimeManagerLogger) Infof(msg string, args ...any) { func (r runtimeManagerLogger) GetContext(ctx context.Context, tool types.Tool, cmd, env []string) (string, []string, error) { return r.rm.GetContext(mvl.WithInfo(ctx, r), tool, cmd, env) } - -func (r runtimeManagerLogger) EnsureCredentialHelpers(ctx context.Context) error { - return r.rm.EnsureCredentialHelpers(mvl.WithInfo(ctx, r)) -} - -func (r runtimeManagerLogger) SetUpCredentialHelpers(_ context.Context, _ *config.CLIConfig) error { - panic("not implemented") -} diff --git a/pkg/sdkserver/credentials.go b/pkg/sdkserver/credentials.go index b0246621..2b527b2b 100644 --- a/pkg/sdkserver/credentials.go +++ b/pkg/sdkserver/credentials.go @@ -7,25 +7,12 @@ import ( "net/http" "slices" - "github.com/gptscript-ai/gptscript/pkg/config" gcontext "github.com/gptscript-ai/gptscript/pkg/context" "github.com/gptscript-ai/gptscript/pkg/credentials" ) -func (s *server) initializeCredentialStore(ctx context.Context, credCtxs []string) (credentials.CredentialStore, error) { - cfg, err := config.ReadCLIConfig(s.gptscriptOpts.OpenAI.ConfigFile) - if err != nil { - return nil, fmt.Errorf("failed to read CLI config: %w", err) - } - - if err := s.runtimeManager.SetUpCredentialHelpers(ctx, cfg); err != nil { - return nil, fmt.Errorf("failed to set up credential helpers: %w", err) - } - if err := s.runtimeManager.EnsureCredentialHelpers(ctx); err != nil { - return nil, fmt.Errorf("failed to ensure credential helpers: %w", err) - } - - store, err := credentials.NewStore(cfg, s.runtimeManager, credCtxs, s.gptscriptOpts.Cache.CacheDir) +func (s *server) initializeCredentialStore(_ context.Context, credCtxs []string) (credentials.CredentialStore, error) { + store, err := s.client.CredentialStoreFactory.NewStore(credCtxs) if err != nil { return nil, fmt.Errorf("failed to initialize credential store: %w", err) } diff --git a/pkg/types/tool.go b/pkg/types/tool.go index cefbd311..e6cbf37f 100644 --- a/pkg/types/tool.go +++ b/pkg/types/tool.go @@ -141,6 +141,7 @@ type Parameters struct { OutputFilters []string `json:"outputFilters,omitempty"` ExportOutputFilters []string `json:"exportOutputFilters,omitempty"` Blocking bool `json:"-"` + Stdin bool `json:"stdin,omitempty"` Type ToolType `json:"type,omitempty"` } @@ -445,6 +446,9 @@ func (t ToolDef) String() string { if t.Parameters.Cache != nil && !*t.Parameters.Cache { _, _ = fmt.Fprintln(buf, "Cache: false") } + if t.Parameters.Stdin { + _, _ = fmt.Fprintln(buf, "Stdin: true") + } if t.Parameters.Temperature != nil { _, _ = fmt.Fprintf(buf, "Temperature: %f\n", *t.Parameters.Temperature) }