feat: add llm-ls config file

crates/llm-ls/Cargo.toml

@@ -7,10 +7,11 @@ edition = "2021"
 name = "llm-ls"
 
 [dependencies]
-candle = { version = "0.3", package = "candle-core", default-features = false }
-candle-nn = "0.3"
-candle-transformers = "0.3"
+candle = { version = "0.4", package = "candle-core", default-features = false }
+candle-nn = "0.4"
+candle-transformers = "0.4"
 clap = { version = "4", features = ["derive"] }
+config = { version = "0.14", features = ["yaml"], default_features = false }
 custom-types = { path = "../custom-types" }
 futures-util = "0.3"
 gitignore = { path = "../gitignore" }
@@ -25,6 +26,7 @@ reqwest = { version = "0.11", default-features = false, features = [
   "rustls-tls",
 ] }
 serde = { version = "1", features = ["derive"] }
+serde_yaml = "0.9"
 serde_json = "1"
 thiserror = "1"
 tinyvec-embed = { path = "../tinyvec-embed" }

crates/llm-ls/src/config.rs

@@ -0,0 +1,36 @@
+use std::path::Path;
+
+use config::Config;
+use serde::{Deserialize, Serialize};
+use tokio::fs::write;
+
+use crate::error::Result;
+
+#[derive(Deserialize, Serialize)]
+pub(crate) struct LlmLsConfig {
+    /// .gitignore-like glob patterns to exclude from indexing
+    pub(crate) ignored_paths: Vec<String>,
+}
+
+impl Default for LlmLsConfig {
+    fn default() -> Self {
+        Self {
+            ignored_paths: vec![".git/".into(), ".idea/".into(), ".DS_Store/".into()],
+        }
+    }
+}
+
+pub async fn load_config(cache_path: &str) -> Result<LlmLsConfig> {
+    let config_file_path = Path::new(cache_path).join("config.yaml");
+    if config_file_path.exists() {
+        Ok(Config::builder()
+            .add_source(config::File::with_name(&format!("{cache_path}/config")))
+            .add_source(config::Environment::with_prefix("LLM_LS"))
+            .build()?
+            .try_deserialize()?)
+    } else {
+        let config = LlmLsConfig::default();
+        write(config_file_path, serde_yaml::to_string(&config)?.as_bytes()).await?;
+        Ok(config)
+    }
+}

crates/llm-ls/src/error.rs

@@ -17,6 +17,8 @@ pub(crate) fn internal_error<E: Display>(err: E) -> LspError {
 pub enum Error {
     #[error("candle error: {0}")]
     Candle(#[from] candle::Error),
+    #[error("config error: {0}")]
+    Config(#[from] config::ConfigError),
     #[error("gitignore error: {0}")]
     Gitignore(#[from] gitignore::Error),
     #[error("huggingface api error: {0}")]
@@ -69,6 +71,8 @@ pub enum Error {
     TokioJoin(#[from] tokio::task::JoinError),
     #[error("unknown backend: {0}")]
     UnknownBackend(String),
+    #[error("yaml serialization error: {0}")]
+    Yaml(#[from] serde_yaml::Error),
 }
 
 pub(crate) type Result<T> = std::result::Result<T, Error>;

crates/llm-ls/src/main.rs

@@ -1,4 +1,5 @@
 use clap::Parser;
+use config::{load_config, LlmLsConfig};
 use custom_types::llm_ls::{
     AcceptCompletionParams, Backend, Completion, FimParams, GetCompletionsParams,
     GetCompletionsResult, Ide, RejectCompletionParams, TokenizerConfig,
@@ -34,6 +35,7 @@ use crate::document::Document;
 use crate::error::{internal_error, Error, Result};
 
 mod backend;
+mod config;
 mod document;
 mod error;
 mod language_id;
@@ -137,6 +139,7 @@ pub struct Generation {
 struct LlmService {
     cache_dir: PathBuf,
     client: Client,
+    config: Arc<LlmLsConfig>,
     document_map: Arc<RwLock<HashMap<String, Document>>>,
     http_client: reqwest::Client,
     unsafe_http_client: reqwest::Client,
@@ -643,6 +646,7 @@ impl LanguageServer for LlmService {
 
     async fn initialized(&self, _: InitializedParams) {
         let client = self.client.clone();
+        let config = self.config.clone();
         let snippet_retriever = self.snippet_retriever.clone();
         let supports_progress_bar = self.supports_progress_bar.clone();
         let workspace_folders = self.workspace_folders.clone();
@@ -683,6 +687,7 @@ impl LanguageServer for LlmService {
                 tokio::select! {
                     res = guard.build_workspace_snippets(
                         client.clone(),
+                        config,
                         token,
                         workspace_folders[0].uri.path(),
                     ) => {
@@ -884,9 +889,19 @@ async fn main() {
             .await
             .expect("failed to initialise snippet retriever"),
     ));
+    let config = Arc::new(
+        load_config(
+            cache_dir
+                .to_str()
+                .expect("cache dir path is not valid utf8"),
+        )
+        .await
+        .expect("failed to load config file"),
+    );
     let (service, socket) = LspService::build(|client| LlmService {
         cache_dir,
         client,
+        config,
         document_map: Arc::new(RwLock::new(HashMap::new())),
         http_client,
         unsafe_http_client,

crates/llm-ls/src/retrieval.rs

@@ -1,3 +1,4 @@
+use crate::config::LlmLsConfig;
 use crate::error::{Error, Result};
 use candle::utils::{cuda_is_available, metal_is_available};
 use candle::{Device, Tensor};
@@ -258,13 +259,21 @@ impl SnippetRetriever {
     pub(crate) async fn build_workspace_snippets(
         &mut self,
         client: Client,
+        config: Arc<LlmLsConfig>,
         token: NumberOrString,
         workspace_root: &str,
     ) -> Result<()> {
         debug!("building workspace snippets");
         let workspace_root = PathBuf::from(workspace_root);
         let mut files = Vec::new();
-        let gitignore = Gitignore::parse(&workspace_root).ok();
+        let mut gitignore = Gitignore::parse(&workspace_root).ok();
+        for pattern in config.ignored_paths.iter() {
+            if let Some(gitignore) = gitignore.as_mut() {
+                if let Err(err) = gitignore.add_rule(pattern.clone()) {
+                    error!("failed to parse pattern: {err}");
+                }
+            };
+        }
 
         client
             .send_notification::<Progress>(ProgressParams {