test tokenize

.gitignore

@@ -2,4 +2,7 @@
 */tmp
 scratch.rs
 data_prep
-*.csv
+*.csv
+.DS_Store
+profile.json
+/scratch

src/lib.rs

@@ -137,6 +137,8 @@ pub fn tokenize(address: &str) -> Vec<String> {
     let address: String = clean_address(address);
 
     address
+        .replace('&', " & ")
+        .replace('#', " # ")
         .split([' ', ',', ';', ')', '\n'].as_ref())
         .filter(|x| !x.is_empty())
         .map(|s| s.to_string())

tests/test_tokenize.rs

@@ -0,0 +1,22 @@
+use us_addrs::tokenize;
+
+#[test]
+fn test_tokenizing() {
+    let mut tokens = tokenize("# 1 abc st");
+    assert_eq!(tokens, vec!["#", "1", "abc", "st"]);
+
+    tokens = tokenize("#1 abc st");
+    assert_eq!(tokens, vec!["#", "1", "abc", "st"]);
+
+    tokens = tokenize("box # 1 abc st");
+    assert_eq!(tokens, vec!["box", "#", "1", "abc", "st"]);
+
+    tokens = tokenize("box #1 abc st");
+    assert_eq!(tokens, vec!["box", "#", "1", "abc", "st"]);
+
+    tokens = tokenize("box# 1 abc st");
+    assert_eq!(tokens, vec!["box", "#", "1", "abc", "st"]);
+
+    tokens = tokenize("box#1 abc st");
+    assert_eq!(tokens, vec!["box", "#", "1", "abc", "st"]);
+}