Merge pull request #56 from IAPark/dependabot/cargo/cargo-d67f63c951

Bump the cargo group with 2 updates

ext/tiktoken_ruby/Cargo.toml

@@ -10,6 +10,6 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = { version = "0.6.1" }
+magnus = { version = "0.7.1" }
 rb-sys = { version = "0.9.87", features = ["stable-api-compiled-fallback"] }
-tiktoken-rs = { version = "0.5.9" }
+tiktoken-rs = { version = "0.6.0" }

ext/tiktoken_ruby/src/core_bpe_wrapper.rs

@@ -1,7 +1,8 @@
 use std::collections::HashSet;
 
-use crate::uncicode_error;
+use tiktoken_rs::Rank;
 
+use crate::uncicode_error;
 
 #[magnus::wrap(class = "Tiktoken::Ext::CoreBPE")]
 pub struct CoreBPEWrapper {
@@ -13,32 +14,34 @@ impl CoreBPEWrapper {
         Self { core_bpe }
     }
 
-    pub fn encode_ordinary(&self, text: String) -> Vec<usize> {
+    pub fn encode_ordinary(&self, text: String) -> Vec<Rank> {
         self.core_bpe.encode_ordinary(text.as_str())
     }
 
-    pub fn encode(&self, text: String, allowed_special: magnus::RArray) -> Result<Vec<usize>, magnus::Error> {
+    pub fn encode(
+        &self,
+        text: String,
+        allowed_special: magnus::RArray,
+    ) -> Result<Vec<Rank>, magnus::Error> {
         let allowed_special: Vec<String> = allowed_special.to_vec()?;
         let allowed_special: Vec<&str> = allowed_special.iter().map(|s| s.as_str()).collect();
         let allowed_special: HashSet<&str> = HashSet::from_iter(allowed_special.iter().cloned());
 
         Ok(self.core_bpe.encode(text.as_str(), allowed_special))
     }
 
-    pub fn encode_with_special_tokens(&self, text: String) -> Vec<usize> {
+    pub fn encode_with_special_tokens(&self, text: String) -> Vec<Rank> {
         self.core_bpe.encode_with_special_tokens(text.as_str())
     }
 
-    pub fn decode(&self, ids: Vec<usize>) -> Result<String, magnus::Error> {
-        self.core_bpe.decode(ids)
-            .map_err(|e| {
-                let error = match uncicode_error() {
-                    Ok(error) => error,
-                    Err(e) => return e
-                };
-
-                magnus::Error::new(error, e.to_string())
-        })
+    pub fn decode(&self, ids: Vec<Rank>) -> Result<String, magnus::Error> {
+        self.core_bpe.decode(ids).map_err(|e| {
+            let error = match uncicode_error() {
+                Ok(error) => error,
+                Err(e) => return e,
+            };
 
+            magnus::Error::new(error, e.to_string())
+        })
     }
 }