test split on punc and spaces

raphaellaude · Nov 14, 2023 · d5109d3 · d5109d3
1 parent 31ac81b
commit d5109d3
Showing 1 changed file with 31 additions and 0 deletions.
diff --git a/tests/test_tokenize.rs b/tests/test_tokenize.rs
@@ -1,3 +1,5 @@
+use std::vec;
+
 use us_addrs::tokenize;
 
 #[test]
@@ -20,3 +22,32 @@ fn test_tokenizing() {
     tokens = tokenize("box#1 abc st");
     assert_eq!(tokens, vec!["box", "#", "1", "abc", "st"]);
 }
+
+#[test]
+fn test_split_on_punc() {
+    // let mut tokens = tokenize("1 abc st,suite 1");
+    // assert_eq!(tokens, vec!["1", "abc", "st,", "suite", "1"]);
+
+    // tokens = tokenize("1 abc st;suite 1");
+    // assert_eq!(tokens, vec!["1", "abc", "st;", "suite", "1"]);
+
+    let tokens = tokenize("1-5 abc road");
+    assert_eq!(tokens, vec!["1-5", "abc", "road"]);
+}
+
+#[test]
+fn test_spaces() {
+    let result = vec!["1", "abc", "st"];
+
+    let mut tokens = tokenize("1 abc st");
+    assert_eq!(tokens, result);
+
+    tokens = tokenize("1 abc  st");
+    assert_eq!(tokens, result);
+
+    tokens = tokenize("1 abc st ");
+    assert_eq!(tokens, result);
+
+    tokens = tokenize(" 1 abc st");
+    assert_eq!(tokens, result);
+}