From 224dff42faac7a4961af7d78a4491fbfcb659d4f Mon Sep 17 00:00:00 2001
From: "Liz3 (Yann HN)" <accs@liz3.net>
Date: Wed, 18 Sep 2024 03:09:06 +0200
Subject: [PATCH 01/31] wip: utf-8 support in strings

---
 src/vm/datatypes/strings.c   |  223 +++--
 src/vm/object.c              |    2 +
 src/vm/object.h              |    1 +
 src/vm/utf8.h                | 1712 ++++++++++++++++++++++++++++++++++
 tests/datetime/import.du     |    2 +-
 tests/strings/find.du        |    7 +-
 tests/strings/isLower.du     |    7 +
 tests/strings/isUpper.du     |    6 +
 tests/strings/len.du         |    5 +-
 tests/strings/split.du       |    6 +
 tests/strings/startsWith.du  |    4 +
 tests/uuid/generate.du       |    2 +-
 tests/uuid/generateRandom.du |    2 +-
 tests/uuid/generateTime.du   |    2 +-
 14 files changed, 1880 insertions(+), 101 deletions(-)
 create mode 100644 src/vm/utf8.h

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index e58ca3e6d..fcb05bf80 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -1,5 +1,11 @@
 #include "strings.h"
 #include "../memory.h"
+#include "../utf8.h"
+#include <stdint.h>
+#include <string.h>
+#include <wctype.h>
+#include <locale.h>
+
 
 
 static Value lenString(DictuVM *vm, int argCount, Value *args) {
@@ -8,6 +14,16 @@ static Value lenString(DictuVM *vm, int argCount, Value *args) {
         return EMPTY_VAL;
     }
 
+    ObjString *string = AS_STRING(args[0]);
+    return NUMBER_VAL(string->character_len);
+}
+
+static Value byteLenString(DictuVM *vm, int argCount, Value *args) {
+    if (argCount != 0) {
+        runtimeError(vm, "len() takes no arguments (%d given)", argCount);
+        return EMPTY_VAL;
+    }
+
     ObjString *string = AS_STRING(args[0]);
     return NUMBER_VAL(string->length);
 }
@@ -64,18 +80,17 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
             replaceStrings[j - 1] = str;
         }
 
-        length += strlen(replaceStrings[j - 1]);
+        length += utf8size_lazy(replaceStrings[j - 1]);
     }
 
     ObjString *string = AS_STRING(args[0]);
 
     int stringLen = string->length + 1;
-    char *tmp = ALLOCATE(vm, char, stringLen);
+    char *tmp = string->chars;
     char *tmpFree = tmp;
-    memcpy(tmp, string->chars, stringLen);
 
     int count = 0;
-    while ((tmp = strstr(tmp, "{}"))) {
+    while ((tmp = utf8str(tmp, "{}"))) {
         count++;
         tmp++;
     }
@@ -100,12 +115,10 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
     int stringLength = 0;
 
     for (int i = 0; i < argCount; ++i) {
-        pos = strstr(tmp, "{}");
-        if (pos != NULL)
-            *pos = '\0';
+        pos = utf8str(tmp, "{}");
 
-        int tmpLength = strlen(tmp);
-        int replaceLength = strlen(replaceStrings[i]);
+        int tmpLength = pos - tmp;
+        int replaceLength = utf8size_lazy(replaceStrings[i]);
         memcpy(newStr + stringLength, tmp, tmpLength);
         memcpy(newStr + stringLength + tmpLength, replaceStrings[i], replaceLength);
         stringLength += tmpLength + replaceLength;
@@ -114,9 +127,8 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
     }
 
     FREE_ARRAY(vm, char*, replaceStrings, argCount);
-    memcpy(newStr + stringLength, tmp, strlen(tmp));
+    memcpy(newStr + stringLength, tmp, utf8size_lazy(tmp));
     newStr[fullLength - 1] = '\0';
-    FREE_ARRAY(vm, char, tmpFree, stringLen);
 
     return OBJ_VAL(takeString(vm, newStr, fullLength - 1));
 }
@@ -147,43 +159,44 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
         }
     }
 
-    char *tmp = ALLOCATE(vm, char, string->length + 1);
-    char *tmpFree = tmp;
-    memcpy(tmp, string->chars, string->length);
-    tmp[string->length] = '\0';
-    int delimiterLength = strlen(delimiter);
+    char *tmp = string->chars;
+    int delimiterLength = utf8size_lazy(delimiter);
     char *token;
 
     ObjList *list = newList(vm);
     push(vm, OBJ_VAL(list));
     int count = 0;
-
+    char* ptr = string->chars;
+    utf8_int32_t cp;
     if (delimiterLength == 0) {
         int tokenIndex = 0;
-        for (; tokenIndex < string->length && count < maxSplit; tokenIndex++) {
+        for (; tokenIndex < string->character_len && count < maxSplit; tokenIndex++) {
             count++;
-            *(tmp) = string->chars[tokenIndex];
-            *(tmp + 1) = '\0';
-            Value str = OBJ_VAL(copyString(vm, tmp, 1));
+          
+            char* next_ptr = utf8codepoint(ptr, &cp);
+
+            // *(tmp) = string->chars[tokenIndex];
+            // *(tmp + 1) = '\0';
+            size_t size = next_ptr - ptr;
+            Value str = OBJ_VAL(copyString(vm, ptr, size));
             // Push to stack to avoid GC
             push(vm, str);
             writeValueArray(vm, &list->values, str);
             pop(vm);
+            ptr = next_ptr;
         }
 
-        if (tokenIndex != string->length && count >= maxSplit) {
-            tmp = (string->chars) + tokenIndex;
+        if (tokenIndex != string->character_len && count >= maxSplit) {
+            tmp = ptr;
         } else {
             tmp = NULL;
         }
     } else if (maxSplit > 0) {
         do {
             count++;
-            token = strstr(tmp, delimiter);
-            if (token)
-                *token = '\0';
+            token = utf8str(tmp, delimiter);
 
-            Value str = OBJ_VAL(copyString(vm, tmp, strlen(tmp)));
+            Value str = OBJ_VAL(copyString(vm, tmp, token == NULL ? utf8size_lazy(tmp) : (token-tmp)));
 
             // Push to stack to avoid GC
             push(vm, str);
@@ -199,7 +212,7 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
     }
 
     if (tmp != NULL && count >= maxSplit) {
-        Value remainingStr = OBJ_VAL(copyString(vm, tmp, strlen(tmp)));
+        Value remainingStr = OBJ_VAL(copyString(vm, tmp, utf8size_lazy(tmp)));
 
         // Push to stack to avoid GC
         push(vm, remainingStr);
@@ -209,7 +222,6 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
 
     pop(vm);
 
-    FREE_ARRAY(vm, char, tmpFree, string->length + 1);
     return OBJ_VAL(list);
 }
 
@@ -227,7 +239,7 @@ static Value containsString(DictuVM *vm, int argCount, Value *args) {
     char *string = AS_CSTRING(args[0]);
     char *delimiter = AS_CSTRING(args[1]);
 
-    if (!strstr(string, delimiter)) {
+    if (!utf8str(string, delimiter)) {
         return FALSE_VAL;
     }
 
@@ -262,14 +274,16 @@ static Value findString(DictuVM *vm, int argCount, Value *args) {
     int position = 0;
 
     for (int i = 0; i < index; ++i) {
-        char *result = strstr(string, substr);
+        if(i != 0) {
+            position +=  utf8len(substr);
+        }
+        char *result = utf8str(string, substr);
         if (!result) {
             position = -1;
             break;
         }
-
-        position += (result - string) + (i * strlen(substr));
-        string = result + strlen(substr);
+        position += utf8nlen(string, (result-string));
+        string = result + utf8size_lazy(substr);
     }
 
     return NUMBER_VAL(position);
@@ -338,17 +352,14 @@ static Value replaceString(DictuVM *vm, int argCount, Value *args) {
     int count = 0;
     int len = to_replace->length;
     int replaceLen = replace->length;
-    int stringLen = strlen(string) + 1;
 
     // Make a copy of the string so we do not modify the original
-    char *tmp = ALLOCATE(vm, char, stringLen + 1);
+    char *tmp = string;
     char *tmpFree = tmp;
-    memcpy(tmp, string, stringLen);
-    tmp[stringLen] = '\0';
 
     // Count the occurrences of the needle so we can determine the size
     // of the string we need to allocate
-    while ((tmp = strstr(tmp, to_replace->chars)) != NULL) {
+    while ((tmp = utf8str(tmp, to_replace->chars)) != NULL) {
         count++;
         tmp += len;
     }
@@ -357,29 +368,25 @@ static Value replaceString(DictuVM *vm, int argCount, Value *args) {
     tmp = tmpFree;
 
     if (count == 0) {
-        FREE_ARRAY(vm, char, tmpFree, stringLen + 1);
         return stringValue;
     }
 
-    int length = strlen(tmp) - count * (len - replaceLen) + 1;
+    int length = utf8size_lazy(tmp) - count * (len - replaceLen) + 1;
     char *pos;
     char *newStr = ALLOCATE(vm, char, length);
     int stringLength = 0;
 
     for (int i = 0; i < count; ++i) {
-        pos = strstr(tmp, to_replace->chars);
-        if (pos != NULL)
-            *pos = '\0';
+        pos = utf8str(tmp, to_replace->chars);
 
-        int tmpLength = strlen(tmp);
+        int tmpLength = pos - tmp;
         memcpy(newStr + stringLength, tmp, tmpLength);
         memcpy(newStr + stringLength + tmpLength, replace->chars, replaceLen);
         stringLength += tmpLength + replaceLen;
         tmp = pos + len;
     }
 
-    memcpy(newStr + stringLength, tmp, strlen(tmp));
-    FREE_ARRAY(vm, char, tmpFree, stringLen + 1);
+    memcpy(newStr + stringLength, tmp, utf8size_lazy(tmp));
     newStr[length - 1] = '\0';
 
     return OBJ_VAL(takeString(vm, newStr, length - 1));
@@ -393,10 +400,8 @@ static Value lowerString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     char *temp = ALLOCATE(vm, char, string->length + 1);
-
-    for (int i = 0; string->chars[i]; i++) {
-        temp[i] = tolower(string->chars[i]);
-    }
+    memcpy(temp, string->chars, string->length);
+    utf8lwr(temp);
     temp[string->length] = '\0';
 
     return OBJ_VAL(takeString(vm, temp, string->length));
@@ -410,11 +415,9 @@ static Value upperString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     char *temp = ALLOCATE(vm, char, string->length + 1);
-
-    for (int i = 0; string->chars[i]; i++) {
-        temp[i] = toupper(string->chars[i]);
-    }
-    temp[string->length] = '\0';
+    memcpy(temp, string->chars, string->length);
+    temp[string->length] = 0x00;
+    utf8upr(temp);
 
     return OBJ_VAL(takeString(vm, temp, string->length));
 }
@@ -430,10 +433,12 @@ static Value startsWithString(DictuVM *vm, int argCount, Value *args) {
         return EMPTY_VAL;
     }
 
-    char *string = AS_CSTRING(args[0]);
+    ObjString *string = AS_STRING(args[0]);
     ObjString *start = AS_STRING(args[1]);
+    if(string->length < start->length)
+        return BOOL_VAL(false);
 
-    return BOOL_VAL(strncmp(string, start->chars, start->length) == 0);
+    return BOOL_VAL(utf8ncmp(string->chars, start->chars, start->length) == 0);
 }
 
 static Value endsWithString(DictuVM *vm, int argCount, Value *args) {
@@ -454,7 +459,7 @@ static Value endsWithString(DictuVM *vm, int argCount, Value *args) {
         return FALSE_VAL;
     }
 
-    return BOOL_VAL(strcmp(string->chars + (string->length - suffix->length), suffix->chars) == 0);
+    return BOOL_VAL(utf8cmp(string->chars + (string->length - suffix->length), suffix->chars) == 0);
 }
 
 static Value leftStripString(DictuVM *vm, int argCount, Value *args) {
@@ -537,7 +542,7 @@ static Value countString(DictuVM *vm, int argCount, Value *args) {
     char *needle = AS_CSTRING(args[1]);
 
     int count = 0;
-    while ((haystack = strstr(haystack, needle))) {
+    while ((haystack = utf8str(haystack, needle))) {
         count++;
         haystack++;
     }
@@ -551,16 +556,19 @@ static Value wordCountString(DictuVM *vm, int argCount, Value *args) {
         return EMPTY_VAL;
     }
 
-    char *string = AS_CSTRING(args[0]);
+    ObjString *string = AS_STRING(args[0]);
     
     int count = 0;
-    int len = strlen(string);
+    int len = string->character_len;
     bool in = false;
+    char* ptr = string->chars;
 
     for (int i = 0; i < len; i++) {
-        if (isspace(string[i])) {
+        utf8_int32_t cp;
+        ptr = utf8codepoint(ptr, &cp);
+        if (iswspace(cp)) {
             in = false;
-        } else if(isalpha(string[i])) {
+        } else if(iswalpha(cp)) {
             if(!in) {
                 in = true;
                 count++;
@@ -579,19 +587,25 @@ static Value titleString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     char *temp = ALLOCATE(vm, char, string->length + 1);
-
+    memcpy(temp, string->chars, string->length);
+    char* ptr = temp;
     bool convertNext=true;
 
-    for (int i = 0; string->chars[i]; i++) {
-        if(string->chars[i]==' '){
+    for (int i = 0; i < string->character_len; i++) {
+        utf8_int32_t cp;
+        char* next_ptr = utf8codepoint(ptr, &cp);
+        if(cp==' '){
             convertNext=true;
         }
         else if(convertNext){
-            temp[i] = toupper(string->chars[i]);
+            cp = utf8uprcodepoint(cp);
             convertNext=false;
-            continue;
+          
+        } else {
+            cp = utf8lwrcodepoint(cp);
         }
-        temp[i] = tolower(string->chars[i]);
+        utf8catcodepoint(ptr, cp, next_ptr -ptr);
+        ptr = next_ptr;
     }
 
     temp[string->length] = '\0';
@@ -616,9 +630,9 @@ static Value repeatString(DictuVM *vm, int argCount, Value *args) {
     int tempLen = (string->length * count) + 1;
     char *temp = ALLOCATE(vm, char, tempLen);
 
-    strcpy(temp, string->chars);
+    utf8cpy(temp, string->chars);
     while (--count > 0) {
-        strcat(temp, string->chars);
+        utf8cat(temp, string->chars);
     }
 
     return OBJ_VAL(takeString(vm, temp, tempLen - 1));
@@ -631,14 +645,16 @@ static Value isUpperString(DictuVM *vm, int argCount, Value *args) {
     }
 
     char *string = AS_CSTRING(args[0]);
-    int len = strlen(string);
+    int len = utf8len(string);
 
     if (len == 0) {
         return BOOL_VAL(false);
     }
 
     for (int i = 0; i < len; i++) {
-        if (!isupper(string[i]) && isalpha(string[i])) {
+        utf8_int32_t cp;
+        string = utf8codepoint(string, &cp);
+        if (!iswupper(cp) && iswalpha(cp)) {
             return BOOL_VAL(false);
         }
     }
@@ -653,14 +669,17 @@ static Value isLowerString(DictuVM *vm, int argCount, Value *args) {
     }
 
     char *string = AS_CSTRING(args[0]);
-    int len = strlen(string);
+    int len = utf8len(string);
 
     if (len == 0) {
         return BOOL_VAL(false);
     }
 
     for (int i = 0; i < len; i++) {
-        if (!islower(string[i]) && isalpha(string[i])) {
+         utf8_int32_t cp;
+        string = utf8codepoint(string, &cp);
+     
+        if (!iswlower(cp) && iswalpha(cp)) {
             return BOOL_VAL(false);
         }
     }
@@ -675,23 +694,30 @@ static Value collapseSpacesString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    char *temp = ALLOCATE(vm, char, string->length + 1);
-    strcpy(temp, string->chars);
-
+    char *dest = ALLOCATE(vm, char, string->length + 1);
+    char* ptr = string->chars;
+    int targetLen = 0;
     int i, j;
-    for (i = j = 0; temp[i]; ++i) {
-        if (!isspace(temp[i]) || (i > 0 && !isspace(temp[i-1]))) {
-            temp[j++] = temp[i];
+    utf8_int32_t last;
+    for (i = j = 0; i < string->character_len; ++i) {
+        utf8_int32_t cp;
+        char* n = utf8codepoint(ptr, &cp);
+        
+        if (!isspace(cp) || (i > 0 && !isspace(last))) {
+            utf8catcodepoint(dest+targetLen, cp, string->length - targetLen);
+            targetLen += utf8codepointsize(cp);
         }
-    }
 
-    temp[j+1] = '\0';
+        last = cp;
+        ptr = n;
+    }
 
-    if (i != j) {
-        temp = SHRINK_ARRAY(vm, temp, char, string->length + 1, j + 1);
+    if (targetLen != string->length) {
+        dest = SHRINK_ARRAY(vm, dest, char, string->length + 1, targetLen + 1);
     }
+    dest[targetLen] = 0x00;
 
-    return OBJ_VAL(takeString(vm, temp, j));
+    return OBJ_VAL(takeString(vm, dest, targetLen));
 }
 
 static Value wrapString(DictuVM *vm, int argCount, Value *args) {
@@ -702,30 +728,37 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     char *temp = ALLOCATE(vm, char, string->length + 1);
-    
+    memcpy(temp, string->chars, string->length);
     int len = AS_NUMBER(args[1]);
 
     int last = 0;
     int count = 0;
-
-    for (int cur = 0; string->chars[cur] != '\0'; cur++, count++) {
-        temp[cur] = string->chars[cur];
-
-        if (isspace(temp[cur])) {
-            last = cur;
+    char*ptr = temp;
+    for (int cur = 0; cur < string->character_len; cur++, count++) {
+        utf8_int32_t cp;
+        char* n = utf8codepoint(ptr, &cp);
+        // temp[cur] = string->chars[cur];
+
+        if (isspace(cp)) {
+            last = ptr - temp;
         } 
 
         if (count >= len) {
             temp[last] = '\n';
             count = 0;
         }
+        ptr = n;
     }
 
-    return OBJ_VAL(takeString(vm, temp, strlen(temp)));
+    return OBJ_VAL(takeString(vm, temp, utf8size_lazy(temp)));
 }
 
 void declareStringMethods(DictuVM *vm) {
+    // Note(Liz3): We need functions from the c stdlib for iswalpha, iswlower, iswupper(the utf8.c 
+    // library functions do not work)
+    setlocale(LC_ALL, "en_US.UTF-8");
     defineNative(vm, &vm->stringMethods, "len", lenString);
+    defineNative(vm, &vm->stringMethods, "byteLen", byteLenString);
     defineNative(vm, &vm->stringMethods, "toNumber", toNumberString);
     defineNative(vm, &vm->stringMethods, "format", formatString);
     defineNative(vm, &vm->stringMethods, "split", splitString);
diff --git a/src/vm/object.c b/src/vm/object.c
index 413b29b8e..587b521c0 100644
--- a/src/vm/object.c
+++ b/src/vm/object.c
@@ -7,6 +7,7 @@
 #include "table.h"
 #include "value.h"
 #include "vm.h"
+#include "utf8.h"
 
 #define ALLOCATE_OBJ(vm, type, objectType) \
     (type*)allocateObject(vm, sizeof(type), objectType)
@@ -152,6 +153,7 @@ static ObjString *allocateString(DictuVM *vm, char *chars, int length,
     string->length = length;
     string->chars = chars;
     string->hash = hash;
+    string->character_len = utf8len(chars);
     push(vm, OBJ_VAL(string));
     tableSet(vm, &vm->strings, string, NIL_VAL);
     pop(vm);
diff --git a/src/vm/object.h b/src/vm/object.h
index 69ff5cc00..1eeb01d7b 100644
--- a/src/vm/object.h
+++ b/src/vm/object.h
@@ -131,6 +131,7 @@ struct sObjString {
     int length;
     char *chars;
     uint32_t hash;
+    int character_len;
 };
 
 struct sObjList {
diff --git a/src/vm/utf8.h b/src/vm/utf8.h
new file mode 100644
index 000000000..495e259db
--- /dev/null
+++ b/src/vm/utf8.h
@@ -0,0 +1,1712 @@
+/* The latest version of this library is available on GitHub;
+ * https://github.com/sheredom/utf8.h */
+
+/* This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * For more information, please refer to <http://unlicense.org/> */
+
+#ifndef SHEREDOM_UTF8_H_INCLUDED
+#define SHEREDOM_UTF8_H_INCLUDED
+
+#if defined(_MSC_VER)
+#pragma warning(push)
+
+/* disable warning: no function prototype given: converting '()' to '(void)' */
+#pragma warning(disable : 4255)
+
+/* disable warning: '__cplusplus' is not defined as a preprocessor macro,
+ * replacing with '0' for '#if/#elif' */
+#pragma warning(disable : 4668)
+
+/* disable warning: bytes padding added after construct */
+#pragma warning(disable : 4820)
+#endif
+
+#if defined(__cplusplus)
+#if defined(_MSC_VER)
+#define utf8_cplusplus _MSVC_LANG
+#else
+#define utf8_cplusplus __cplusplus
+#endif
+#endif
+
+#include <stddef.h>
+#include <stdlib.h>
+
+#if defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+
+#if defined(_MSC_VER) && (_MSC_VER < 1920)
+typedef __int32 utf8_int32_t;
+#else
+#include <stdint.h>
+typedef int32_t utf8_int32_t;
+#endif
+
+#if defined(__clang__)
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wold-style-cast"
+#pragma clang diagnostic ignored "-Wcast-qual"
+
+#if __has_warning("-Wunsafe-buffer-usage")
+#pragma clang diagnostic ignored "-Wunsafe-buffer-usage"
+#endif
+#endif
+
+#ifdef utf8_cplusplus
+extern "C" {
+#endif
+
+#if defined(__TINYC__)
+#define UTF8_ATTRIBUTE(a) __attribute((a))
+#else
+#define UTF8_ATTRIBUTE(a) __attribute__((a))
+#endif
+
+#if defined(_MSC_VER)
+#define utf8_nonnull
+#define utf8_pure
+#define utf8_restrict __restrict
+#define utf8_weak __inline
+#elif defined(__clang__) || defined(__GNUC__)
+#define utf8_nonnull UTF8_ATTRIBUTE(nonnull)
+#define utf8_pure UTF8_ATTRIBUTE(pure)
+#define utf8_restrict __restrict__
+#define utf8_weak UTF8_ATTRIBUTE(weak)
+#elif defined(__TINYC__)
+#define utf8_nonnull UTF8_ATTRIBUTE(nonnull)
+#define utf8_pure UTF8_ATTRIBUTE(pure)
+#define utf8_restrict
+#define utf8_weak UTF8_ATTRIBUTE(weak)
+#else
+#error Non clang, non gcc, non MSVC, non tcc compiler found!
+#endif
+
+#ifdef utf8_cplusplus
+#define utf8_null NULL
+#else
+#define utf8_null 0
+#endif
+
+#if defined(utf8_cplusplus) && utf8_cplusplus >= 201402L && (!defined(_MSC_VER) || (defined(_MSC_VER) && _MSC_VER >= 1910))
+#define utf8_constexpr14 constexpr
+#define utf8_constexpr14_impl constexpr
+#else
+/* constexpr and weak are incompatible. so only enable one of them */
+#define utf8_constexpr14 utf8_weak
+#define utf8_constexpr14_impl
+#endif
+
+#if defined(utf8_cplusplus) && utf8_cplusplus >= 202002L
+using utf8_int8_t = char8_t; /* Introduced in C++20 */
+#else
+typedef char utf8_int8_t;
+#endif
+
+/* Return less than 0, 0, greater than 0 if src1 < src2, src1 == src2, src1 >
+ * src2 respectively, case insensitive. */
+utf8_constexpr14 utf8_nonnull utf8_pure int
+utf8casecmp(const utf8_int8_t *src1, const utf8_int8_t *src2);
+
+/* Append the utf8 string src onto the utf8 string dst. */
+utf8_nonnull utf8_weak utf8_int8_t *
+utf8cat(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src);
+
+/* Find the first match of the utf8 codepoint chr in the utf8 string src. */
+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
+utf8chr(const utf8_int8_t *src, utf8_int32_t chr);
+
+/* Return less than 0, 0, greater than 0 if src1 < src2,
+ * src1 == src2, src1 > src2 respectively. */
+utf8_constexpr14 utf8_nonnull utf8_pure int utf8cmp(const utf8_int8_t *src1,
+                                                    const utf8_int8_t *src2);
+
+/* Copy the utf8 string src onto the memory allocated in dst. */
+utf8_nonnull utf8_weak utf8_int8_t *
+utf8cpy(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src);
+
+/* Number of utf8 codepoints in the utf8 string src that consists entirely
+ * of utf8 codepoints not from the utf8 string reject. */
+utf8_constexpr14 utf8_nonnull utf8_pure size_t
+utf8cspn(const utf8_int8_t *src, const utf8_int8_t *reject);
+
+/* Duplicate the utf8 string src by getting its size, malloc'ing a new buffer
+ * copying over the data, and returning that. Or 0 if malloc failed. */
+utf8_weak utf8_int8_t *utf8dup(const utf8_int8_t *src);
+
+/* Number of utf8 codepoints in the utf8 string str,
+ * excluding the null terminating byte. */
+utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8len(const utf8_int8_t *str);
+
+/* Similar to utf8len, except that only at most n bytes of src are looked. */
+utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8nlen(const utf8_int8_t *str,
+                                                        size_t n);
+
+/* Return less than 0, 0, greater than 0 if src1 < src2, src1 == src2, src1 >
+ * src2 respectively, case insensitive. Checking at most n bytes of each utf8
+ * string. */
+utf8_constexpr14 utf8_nonnull utf8_pure int
+utf8ncasecmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n);
+
+/* Append the utf8 string src onto the utf8 string dst,
+ * writing at most n+1 bytes. Can produce an invalid utf8
+ * string if n falls partway through a utf8 codepoint. */
+utf8_nonnull utf8_weak utf8_int8_t *
+utf8ncat(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src,
+         size_t n);
+
+/* Return less than 0, 0, greater than 0 if src1 < src2,
+ * src1 == src2, src1 > src2 respectively. Checking at most n
+ * bytes of each utf8 string. */
+utf8_constexpr14 utf8_nonnull utf8_pure int
+utf8ncmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n);
+
+/* Copy the utf8 string src onto the memory allocated in dst.
+ * Copies at most n bytes. If n falls partway through a utf8
+ * codepoint, or if dst doesn't have enough room for a null
+ * terminator, the final string will be cut short to preserve
+ * utf8 validity. */
+
+utf8_nonnull utf8_weak utf8_int8_t *
+utf8ncpy(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src,
+         size_t n);
+
+/* Similar to utf8dup, except that at most n bytes of src are copied. If src is
+ * longer than n, only n bytes are copied and a null byte is added.
+ *
+ * Returns a new string if successful, 0 otherwise */
+utf8_weak utf8_int8_t *utf8ndup(const utf8_int8_t *src, size_t n);
+
+/* Locates the first occurrence in the utf8 string str of any byte in the
+ * utf8 string accept, or 0 if no match was found. */
+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
+utf8pbrk(const utf8_int8_t *str, const utf8_int8_t *accept);
+
+/* Find the last match of the utf8 codepoint chr in the utf8 string src. */
+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
+utf8rchr(const utf8_int8_t *src, int chr);
+
+/* Number of bytes in the utf8 string str,
+ * including the null terminating byte. */
+utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8size(const utf8_int8_t *str);
+
+/* Similar to utf8size, except that the null terminating byte is excluded. */
+utf8_constexpr14 utf8_nonnull utf8_pure size_t
+utf8size_lazy(const utf8_int8_t *str);
+
+/* Similar to utf8size, except that only at most n bytes of src are looked and
+ * the null terminating byte is excluded. */
+utf8_constexpr14 utf8_nonnull utf8_pure size_t
+utf8nsize_lazy(const utf8_int8_t *str, size_t n);
+
+/* Number of utf8 codepoints in the utf8 string src that consists entirely
+ * of utf8 codepoints from the utf8 string accept. */
+utf8_constexpr14 utf8_nonnull utf8_pure size_t
+utf8spn(const utf8_int8_t *src, const utf8_int8_t *accept);
+
+/* The position of the utf8 string needle in the utf8 string haystack. */
+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
+utf8str(const utf8_int8_t *haystack, const utf8_int8_t *needle);
+
+/* The position of the utf8 string needle in the utf8 string haystack, case
+ * insensitive. */
+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
+utf8casestr(const utf8_int8_t *haystack, const utf8_int8_t *needle);
+
+/* Return 0 on success, or the position of the invalid
+ * utf8 codepoint on failure. */
+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
+utf8valid(const utf8_int8_t *str);
+
+/* Similar to utf8valid, except that only at most n bytes of src are looked. */
+utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *
+utf8nvalid(const utf8_int8_t *str, size_t n);
+
+/* Given a null-terminated string, makes the string valid by replacing invalid
+ * codepoints with a 1-byte replacement. Returns 0 on success. */
+utf8_nonnull utf8_weak int utf8makevalid(utf8_int8_t *str,
+                                         const utf8_int32_t replacement);
+
+/* Sets out_codepoint to the current utf8 codepoint in str, and returns the
+ * address of the next utf8 codepoint after the current one in str. */
+utf8_constexpr14 utf8_nonnull utf8_int8_t *
+utf8codepoint(const utf8_int8_t *utf8_restrict str,
+              utf8_int32_t *utf8_restrict out_codepoint);
+
+/* Calculates the size of the next utf8 codepoint in str. */
+utf8_constexpr14 utf8_nonnull size_t
+utf8codepointcalcsize(const utf8_int8_t *str);
+
+/* Returns the size of the given codepoint in bytes. */
+utf8_constexpr14 size_t utf8codepointsize(utf8_int32_t chr);
+
+/* Write a codepoint to the given string, and return the address to the next
+ * place after the written codepoint. Pass how many bytes left in the buffer to
+ * n. If there is not enough space for the codepoint, this function returns
+ * null. */
+utf8_nonnull utf8_weak utf8_int8_t *
+utf8catcodepoint(utf8_int8_t *str, utf8_int32_t chr, size_t n);
+
+/* Returns 1 if the given character is lowercase, or 0 if it is not. */
+utf8_constexpr14 int utf8islower(utf8_int32_t chr);
+
+/* Returns 1 if the given character is uppercase, or 0 if it is not. */
+utf8_constexpr14 int utf8isupper(utf8_int32_t chr);
+
+/* Transform the given string into all lowercase codepoints. */
+utf8_nonnull utf8_weak void utf8lwr(utf8_int8_t *utf8_restrict str);
+
+/* Transform the given string into all uppercase codepoints. */
+utf8_nonnull utf8_weak void utf8upr(utf8_int8_t *utf8_restrict str);
+
+/* Make a codepoint lower case if possible. */
+utf8_constexpr14 utf8_int32_t utf8lwrcodepoint(utf8_int32_t cp);
+
+/* Make a codepoint upper case if possible. */
+utf8_constexpr14 utf8_int32_t utf8uprcodepoint(utf8_int32_t cp);
+
+/* Sets out_codepoint to the current utf8 codepoint in str, and returns the
+ * address of the previous utf8 codepoint before the current one in str. */
+utf8_constexpr14 utf8_nonnull utf8_int8_t *
+utf8rcodepoint(const utf8_int8_t *utf8_restrict str,
+               utf8_int32_t *utf8_restrict out_codepoint);
+
+/* Duplicate the utf8 string src by getting its size, calling alloc_func_ptr to
+ * copy over data to a new buffer, and returning that. Or 0 if alloc_func_ptr
+ * returned null. */
+utf8_weak utf8_int8_t *utf8dup_ex(const utf8_int8_t *src,
+                                  utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *,
+                                                                 size_t),
+                                  utf8_int8_t *user_data);
+
+/* Similar to utf8dup, except that at most n bytes of src are copied. If src is
+ * longer than n, only n bytes are copied and a null byte is added.
+ *
+ * Returns a new string if successful, 0 otherwise. */
+utf8_weak utf8_int8_t *utf8ndup_ex(const utf8_int8_t *src, size_t n,
+                                   utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *,
+                                                                  size_t),
+                                   utf8_int8_t *user_data);
+
+#undef utf8_weak
+#undef utf8_pure
+#undef utf8_nonnull
+
+utf8_constexpr14_impl int utf8casecmp(const utf8_int8_t *src1,
+                                      const utf8_int8_t *src2) {
+  utf8_int32_t src1_lwr_cp = 0, src2_lwr_cp = 0, src1_upr_cp = 0,
+               src2_upr_cp = 0, src1_orig_cp = 0, src2_orig_cp = 0;
+
+  for (;;) {
+    src1 = utf8codepoint(src1, &src1_orig_cp);
+    src2 = utf8codepoint(src2, &src2_orig_cp);
+
+    /* lower the srcs if required */
+    src1_lwr_cp = utf8lwrcodepoint(src1_orig_cp);
+    src2_lwr_cp = utf8lwrcodepoint(src2_orig_cp);
+
+    /* lower the srcs if required */
+    src1_upr_cp = utf8uprcodepoint(src1_orig_cp);
+    src2_upr_cp = utf8uprcodepoint(src2_orig_cp);
+
+    /* check if the lowered codepoints match */
+    if ((0 == src1_orig_cp) && (0 == src2_orig_cp)) {
+      return 0;
+    } else if ((src1_lwr_cp == src2_lwr_cp) || (src1_upr_cp == src2_upr_cp)) {
+      continue;
+    }
+
+    /* if they don't match, then we return the difference between the characters
+     */
+    return src1_lwr_cp - src2_lwr_cp;
+  }
+}
+
+utf8_int8_t *utf8cat(utf8_int8_t *utf8_restrict dst,
+                     const utf8_int8_t *utf8_restrict src) {
+  utf8_int8_t *d = dst;
+  /* find the null terminating byte in dst */
+  while ('\0' != *d) {
+    d++;
+  }
+
+  /* overwriting the null terminating byte in dst, append src byte-by-byte */
+  while ('\0' != *src) {
+    *d++ = *src++;
+  }
+
+  /* write out a new null terminating byte into dst */
+  *d = '\0';
+
+  return dst;
+}
+
+utf8_constexpr14_impl utf8_int8_t *utf8chr(const utf8_int8_t *src,
+                                           utf8_int32_t chr) {
+  utf8_int8_t c[5] = {'\0', '\0', '\0', '\0', '\0'};
+
+  if (0 == chr) {
+    /* being asked to return position of null terminating byte, so
+     * just run s to the end, and return! */
+    while ('\0' != *src) {
+      src++;
+    }
+    return (utf8_int8_t *)src;
+  } else if (0 == ((utf8_int32_t)0xffffff80 & chr)) {
+    /* 1-byte/7-bit ascii
+     * (0b0xxxxxxx) */
+    c[0] = (utf8_int8_t)chr;
+  } else if (0 == ((utf8_int32_t)0xfffff800 & chr)) {
+    /* 2-byte/11-bit utf8 code point
+     * (0b110xxxxx 0b10xxxxxx) */
+    c[0] = (utf8_int8_t)(0xc0 | (utf8_int8_t)(chr >> 6));
+    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
+  } else if (0 == ((utf8_int32_t)0xffff0000 & chr)) {
+    /* 3-byte/16-bit utf8 code point
+     * (0b1110xxxx 0b10xxxxxx 0b10xxxxxx) */
+    c[0] = (utf8_int8_t)(0xe0 | (utf8_int8_t)(chr >> 12));
+    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
+    c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
+  } else { /* if (0 == ((int)0xffe00000 & chr)) { */
+    /* 4-byte/21-bit utf8 code point
+     * (0b11110xxx 0b10xxxxxx 0b10xxxxxx 0b10xxxxxx) */
+    c[0] = (utf8_int8_t)(0xf0 | (utf8_int8_t)(chr >> 18));
+    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 12) & 0x3f));
+    c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
+    c[3] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
+  }
+
+  /* we've made c into a 2 utf8 codepoint string, one for the chr we are
+   * seeking, another for the null terminating byte. Now use utf8str to
+   * search */
+  return utf8str(src, c);
+}
+
+utf8_constexpr14_impl int utf8cmp(const utf8_int8_t *src1,
+                                  const utf8_int8_t *src2) {
+  while (('\0' != *src1) || ('\0' != *src2)) {
+    if (*src1 < *src2) {
+      return -1;
+    } else if (*src1 > *src2) {
+      return 1;
+    }
+
+    src1++;
+    src2++;
+  }
+
+  /* both utf8 strings matched */
+  return 0;
+}
+
+utf8_constexpr14_impl int utf8coll(const utf8_int8_t *src1,
+                                   const utf8_int8_t *src2);
+
+utf8_int8_t *utf8cpy(utf8_int8_t *utf8_restrict dst,
+                     const utf8_int8_t *utf8_restrict src) {
+  utf8_int8_t *d = dst;
+
+  /* overwriting anything previously in dst, write byte-by-byte
+   * from src */
+  while ('\0' != *src) {
+    *d++ = *src++;
+  }
+
+  /* append null terminating byte */
+  *d = '\0';
+
+  return dst;
+}
+
+utf8_constexpr14_impl size_t utf8cspn(const utf8_int8_t *src,
+                                      const utf8_int8_t *reject) {
+  size_t chars = 0;
+
+  while ('\0' != *src) {
+    const utf8_int8_t *r = reject;
+    size_t offset = 0;
+
+    while ('\0' != *r) {
+      /* checking that if *r is the start of a utf8 codepoint
+       * (it is not 0b10xxxxxx) and we have successfully matched
+       * a previous character (0 < offset) - we found a match */
+      if ((0x80 != (0xc0 & *r)) && (0 < offset)) {
+        return chars;
+      } else {
+        if (*r == src[offset]) {
+          /* part of a utf8 codepoint matched, so move our checking
+           * onwards to the next byte */
+          offset++;
+          r++;
+        } else {
+          /* r could be in the middle of an unmatching utf8 code point,
+           * so we need to march it on to the next character beginning, */
+
+          do {
+            r++;
+          } while (0x80 == (0xc0 & *r));
+
+          /* reset offset too as we found a mismatch */
+          offset = 0;
+        }
+      }
+    }
+
+    /* found a match at the end of *r, so didn't get a chance to test it */
+    if (0 < offset) {
+      return chars;
+    }
+
+    /* the current utf8 codepoint in src did not match reject, but src
+     * could have been partway through a utf8 codepoint, so we need to
+     * march it onto the next utf8 codepoint starting byte */
+    do {
+      src++;
+    } while ((0x80 == (0xc0 & *src)));
+    chars++;
+  }
+
+  return chars;
+}
+
+utf8_int8_t *utf8dup(const utf8_int8_t *src) {
+  return utf8dup_ex(src, utf8_null, utf8_null);
+}
+
+utf8_int8_t *utf8dup_ex(const utf8_int8_t *src,
+                        utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, size_t),
+                        utf8_int8_t *user_data) {
+  utf8_int8_t *n = utf8_null;
+
+  /* figure out how many bytes (including the terminator) we need to copy first
+   */
+  size_t bytes = utf8size(src);
+
+  if (alloc_func_ptr) {
+    n = alloc_func_ptr(user_data, bytes);
+  } else {
+#if !defined(UTF8_NO_STD_MALLOC)
+    n = (utf8_int8_t *)malloc(bytes);
+#else
+    return utf8_null;
+#endif
+  }
+
+  if (utf8_null == n) {
+    /* out of memory so we bail */
+    return utf8_null;
+  } else {
+    bytes = 0;
+
+    /* copy src byte-by-byte into our new utf8 string */
+    while ('\0' != src[bytes]) {
+      n[bytes] = src[bytes];
+      bytes++;
+    }
+
+    /* append null terminating byte */
+    n[bytes] = '\0';
+    return n;
+  }
+}
+
+utf8_constexpr14_impl utf8_int8_t *utf8fry(const utf8_int8_t *str);
+
+utf8_constexpr14_impl size_t utf8len(const utf8_int8_t *str) {
+  return utf8nlen(str, SIZE_MAX);
+}
+
+utf8_constexpr14_impl size_t utf8nlen(const utf8_int8_t *str, size_t n) {
+  const utf8_int8_t *t = str;
+  size_t length = 0;
+
+  while ((size_t)(str - t) < n && '\0' != *str) {
+    if (0xf0 == (0xf8 & *str)) {
+      /* 4-byte utf8 code point (began with 0b11110xxx) */
+      str += 4;
+    } else if (0xe0 == (0xf0 & *str)) {
+      /* 3-byte utf8 code point (began with 0b1110xxxx) */
+      str += 3;
+    } else if (0xc0 == (0xe0 & *str)) {
+      /* 2-byte utf8 code point (began with 0b110xxxxx) */
+      str += 2;
+    } else { /* if (0x00 == (0x80 & *s)) { */
+      /* 1-byte ascii (began with 0b0xxxxxxx) */
+      str += 1;
+    }
+
+    /* no matter the bytes we marched s forward by, it was
+     * only 1 utf8 codepoint */
+    length++;
+  }
+
+  if ((size_t)(str - t) > n) {
+    length--;
+  }
+  return length;
+}
+
+utf8_constexpr14_impl int utf8ncasecmp(const utf8_int8_t *src1,
+                                       const utf8_int8_t *src2, size_t n) {
+  utf8_int32_t src1_lwr_cp = 0, src2_lwr_cp = 0, src1_upr_cp = 0,
+               src2_upr_cp = 0, src1_orig_cp = 0, src2_orig_cp = 0;
+
+  do {
+    const utf8_int8_t *const s1 = src1;
+    const utf8_int8_t *const s2 = src2;
+
+    /* first check that we have enough bytes left in n to contain an entire
+     * codepoint */
+    if (0 == n) {
+      return 0;
+    }
+
+    if ((1 == n) && ((0xc0 == (0xe0 & *s1)) || (0xc0 == (0xe0 & *s2)))) {
+      const utf8_int32_t c1 = (0xe0 & *s1);
+      const utf8_int32_t c2 = (0xe0 & *s2);
+
+      if (c1 != c2) {
+        return c1 - c2;
+      } else {
+        return 0;
+      }
+    }
+
+    if ((2 >= n) && ((0xe0 == (0xf0 & *s1)) || (0xe0 == (0xf0 & *s2)))) {
+      const utf8_int32_t c1 = (0xf0 & *s1);
+      const utf8_int32_t c2 = (0xf0 & *s2);
+
+      if (c1 != c2) {
+        return c1 - c2;
+      } else {
+        return 0;
+      }
+    }
+
+    if ((3 >= n) && ((0xf0 == (0xf8 & *s1)) || (0xf0 == (0xf8 & *s2)))) {
+      const utf8_int32_t c1 = (0xf8 & *s1);
+      const utf8_int32_t c2 = (0xf8 & *s2);
+
+      if (c1 != c2) {
+        return c1 - c2;
+      } else {
+        return 0;
+      }
+    }
+
+    src1 = utf8codepoint(src1, &src1_orig_cp);
+    src2 = utf8codepoint(src2, &src2_orig_cp);
+    n -= utf8codepointsize(src1_orig_cp);
+
+    src1_lwr_cp = utf8lwrcodepoint(src1_orig_cp);
+    src2_lwr_cp = utf8lwrcodepoint(src2_orig_cp);
+
+    src1_upr_cp = utf8uprcodepoint(src1_orig_cp);
+    src2_upr_cp = utf8uprcodepoint(src2_orig_cp);
+
+    /* check if the lowered codepoints match */
+    if ((0 == src1_orig_cp) && (0 == src2_orig_cp)) {
+      return 0;
+    } else if ((src1_lwr_cp == src2_lwr_cp) || (src1_upr_cp == src2_upr_cp)) {
+      continue;
+    }
+
+    /* if they don't match, then we return the difference between the characters
+     */
+    return src1_lwr_cp - src2_lwr_cp;
+  } while (0 < n);
+
+  /* both utf8 strings matched */
+  return 0;
+}
+
+utf8_int8_t *utf8ncat(utf8_int8_t *utf8_restrict dst,
+                      const utf8_int8_t *utf8_restrict src, size_t n) {
+  utf8_int8_t *d = dst;
+
+  /* find the null terminating byte in dst */
+  while ('\0' != *d) {
+    d++;
+  }
+
+  /* overwriting the null terminating byte in dst, append src byte-by-byte
+   * stopping if we run out of space */
+  while (('\0' != *src) && (0 != n--)) {
+    *d++ = *src++;
+  }
+
+  /* write out a new null terminating byte into dst */
+  *d = '\0';
+
+  return dst;
+}
+
+utf8_constexpr14_impl int utf8ncmp(const utf8_int8_t *src1,
+                                   const utf8_int8_t *src2, size_t n) {
+  while ((0 != n--) && (('\0' != *src1) || ('\0' != *src2))) {
+    if (*src1 < *src2) {
+      return -1;
+    } else if (*src1 > *src2) {
+      return 1;
+    }
+
+    src1++;
+    src2++;
+  }
+
+  /* both utf8 strings matched */
+  return 0;
+}
+
+utf8_int8_t *utf8ncpy(utf8_int8_t *utf8_restrict dst,
+                      const utf8_int8_t *utf8_restrict src, size_t n) {
+  utf8_int8_t *d = dst;
+  size_t index = 0, check_index = 0;
+
+  if (n == 0) {
+    return dst;
+  }
+
+  /* overwriting anything previously in dst, write byte-by-byte
+   * from src */
+  for (index = 0; index < n; index++) {
+    d[index] = src[index];
+    if ('\0' == src[index]) {
+      break;
+    }
+  }
+
+  for (check_index = index - 1;
+       check_index > 0 && 0x80 == (0xc0 & d[check_index]); check_index--) {
+    /* just moving the index */
+  }
+
+  if (check_index < index &&
+      ((index - check_index) < utf8codepointcalcsize(&d[check_index]) ||
+       (index - check_index) == n)) {
+    index = check_index;
+  }
+
+  /* append null terminating byte */
+  for (; index < n; index++) {
+    d[index] = 0;
+  }
+
+  return dst;
+}
+
+utf8_int8_t *utf8ndup(const utf8_int8_t *src, size_t n) {
+  return utf8ndup_ex(src, n, utf8_null, utf8_null);
+}
+
+utf8_int8_t *utf8ndup_ex(const utf8_int8_t *src, size_t n,
+                         utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, size_t),
+                         utf8_int8_t *user_data) {
+  utf8_int8_t *c = utf8_null;
+  size_t bytes = 0;
+
+  /* Find the end of the string or stop when n is reached */
+  while ('\0' != src[bytes] && bytes < n) {
+    bytes++;
+  }
+
+  /* In case bytes is actually less than n, we need to set it
+   * to be used later in the copy byte by byte. */
+  n = bytes;
+
+  if (alloc_func_ptr) {
+    c = alloc_func_ptr(user_data, bytes + 1);
+  } else {
+#if !defined(UTF8_NO_STD_MALLOC)
+    c = (utf8_int8_t *)malloc(bytes + 1);
+#else
+    c = utf8_null;
+#endif
+  }
+
+  if (utf8_null == c) {
+    /* out of memory so we bail */
+    return utf8_null;
+  }
+
+  bytes = 0;
+
+  /* copy src byte-by-byte into our new utf8 string */
+  while ('\0' != src[bytes] && bytes < n) {
+    c[bytes] = src[bytes];
+    bytes++;
+  }
+
+  /* append null terminating byte */
+  c[bytes] = '\0';
+  return c;
+}
+
+utf8_constexpr14_impl utf8_int8_t *utf8rchr(const utf8_int8_t *src, int chr) {
+
+  utf8_int8_t *match = utf8_null;
+  utf8_int8_t c[5] = {'\0', '\0', '\0', '\0', '\0'};
+
+  if (0 == chr) {
+    /* being asked to return position of null terminating byte, so
+     * just run s to the end, and return! */
+    while ('\0' != *src) {
+      src++;
+    }
+    return (utf8_int8_t *)src;
+  } else if (0 == ((int)0xffffff80 & chr)) {
+    /* 1-byte/7-bit ascii
+     * (0b0xxxxxxx) */
+    c[0] = (utf8_int8_t)chr;
+  } else if (0 == ((int)0xfffff800 & chr)) {
+    /* 2-byte/11-bit utf8 code point
+     * (0b110xxxxx 0b10xxxxxx) */
+    c[0] = (utf8_int8_t)(0xc0 | (utf8_int8_t)(chr >> 6));
+    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
+  } else if (0 == ((int)0xffff0000 & chr)) {
+    /* 3-byte/16-bit utf8 code point
+     * (0b1110xxxx 0b10xxxxxx 0b10xxxxxx) */
+    c[0] = (utf8_int8_t)(0xe0 | (utf8_int8_t)(chr >> 12));
+    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
+    c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
+  } else { /* if (0 == ((int)0xffe00000 & chr)) { */
+    /* 4-byte/21-bit utf8 code point
+     * (0b11110xxx 0b10xxxxxx 0b10xxxxxx 0b10xxxxxx) */
+    c[0] = (utf8_int8_t)(0xf0 | (utf8_int8_t)(chr >> 18));
+    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 12) & 0x3f));
+    c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
+    c[3] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
+  }
+
+  /* we've created a 2 utf8 codepoint string in c that is
+   * the utf8 character asked for by chr, and a null
+   * terminating byte */
+
+  while ('\0' != *src) {
+    size_t offset = 0;
+
+    while ((src[offset] == c[offset]) && ('\0' != src[offset])) {
+      offset++;
+    }
+
+    if ('\0' == c[offset]) {
+      /* we found a matching utf8 code point */
+      match = (utf8_int8_t *)src;
+      src += offset;
+
+      if ('\0' == *src) {
+        break;
+      }
+    } else {
+      src += offset;
+
+      /* need to march s along to next utf8 codepoint start
+       * (the next byte that doesn't match 0b10xxxxxx) */
+      if ('\0' != *src) {
+        do {
+          src++;
+        } while (0x80 == (0xc0 & *src));
+      }
+    }
+  }
+
+  /* return the last match we found (or 0 if no match was found) */
+  return match;
+}
+
+utf8_constexpr14_impl utf8_int8_t *utf8pbrk(const utf8_int8_t *str,
+                                            const utf8_int8_t *accept) {
+  while ('\0' != *str) {
+    const utf8_int8_t *a = accept;
+    size_t offset = 0;
+
+    while ('\0' != *a) {
+      /* checking that if *a is the start of a utf8 codepoint
+       * (it is not 0b10xxxxxx) and we have successfully matched
+       * a previous character (0 < offset) - we found a match */
+      if ((0x80 != (0xc0 & *a)) && (0 < offset)) {
+        return (utf8_int8_t *)str;
+      } else {
+        if (*a == str[offset]) {
+          /* part of a utf8 codepoint matched, so move our checking
+           * onwards to the next byte */
+          offset++;
+          a++;
+        } else {
+          /* r could be in the middle of an unmatching utf8 code point,
+           * so we need to march it on to the next character beginning, */
+
+          do {
+            a++;
+          } while (0x80 == (0xc0 & *a));
+
+          /* reset offset too as we found a mismatch */
+          offset = 0;
+        }
+      }
+    }
+
+    /* we found a match on the last utf8 codepoint */
+    if (0 < offset) {
+      return (utf8_int8_t *)str;
+    }
+
+    /* the current utf8 codepoint in src did not match accept, but src
+     * could have been partway through a utf8 codepoint, so we need to
+     * march it onto the next utf8 codepoint starting byte */
+    do {
+      str++;
+    } while ((0x80 == (0xc0 & *str)));
+  }
+
+  return utf8_null;
+}
+
+utf8_constexpr14_impl size_t utf8size(const utf8_int8_t *str) {
+  return utf8size_lazy(str) + 1;
+}
+
+utf8_constexpr14_impl size_t utf8size_lazy(const utf8_int8_t *str) {
+  return utf8nsize_lazy(str, SIZE_MAX);
+}
+
+utf8_constexpr14_impl size_t utf8nsize_lazy(const utf8_int8_t *str, size_t n) {
+  size_t size = 0;
+  while (size < n && '\0' != str[size]) {
+    size++;
+  }
+  return size;
+}
+
+utf8_constexpr14_impl size_t utf8spn(const utf8_int8_t *src,
+                                     const utf8_int8_t *accept) {
+  size_t chars = 0;
+
+  while ('\0' != *src) {
+    const utf8_int8_t *a = accept;
+    size_t offset = 0;
+
+    while ('\0' != *a) {
+      /* checking that if *r is the start of a utf8 codepoint
+       * (it is not 0b10xxxxxx) and we have successfully matched
+       * a previous character (0 < offset) - we found a match */
+      if ((0x80 != (0xc0 & *a)) && (0 < offset)) {
+        /* found a match, so increment the number of utf8 codepoints
+         * that have matched and stop checking whether any other utf8
+         * codepoints in a match */
+        chars++;
+        src += offset;
+        offset = 0;
+        break;
+      } else {
+        if (*a == src[offset]) {
+          offset++;
+          a++;
+        } else {
+          /* a could be in the middle of an unmatching utf8 codepoint,
+           * so we need to march it on to the next character beginning, */
+          do {
+            a++;
+          } while (0x80 == (0xc0 & *a));
+
+          /* reset offset too as we found a mismatch */
+          offset = 0;
+        }
+      }
+    }
+
+    /* found a match at the end of *a, so didn't get a chance to test it */
+    if (0 < offset) {
+      chars++;
+      src += offset;
+      continue;
+    }
+
+    /* if a got to its terminating null byte, then we didn't find a match.
+     * Return the current number of matched utf8 codepoints */
+    if ('\0' == *a) {
+      return chars;
+    }
+  }
+
+  return chars;
+}
+
+utf8_constexpr14_impl utf8_int8_t *utf8str(const utf8_int8_t *haystack,
+                                           const utf8_int8_t *needle) {
+  utf8_int32_t throwaway_codepoint = 0;
+
+  /* if needle has no utf8 codepoints before the null terminating
+   * byte then return haystack */
+  if ('\0' == *needle) {
+    return (utf8_int8_t *)haystack;
+  }
+
+  while ('\0' != *haystack) {
+    const utf8_int8_t *maybeMatch = haystack;
+    const utf8_int8_t *n = needle;
+
+    while (*haystack == *n && (*haystack != '\0' && *n != '\0')) {
+      n++;
+      haystack++;
+    }
+
+    if ('\0' == *n) {
+      /* we found the whole utf8 string for needle in haystack at
+       * maybeMatch, so return it */
+      return (utf8_int8_t *)maybeMatch;
+    } else {
+      /* h could be in the middle of an unmatching utf8 codepoint,
+       * so we need to march it on to the next character beginning
+       * starting from the current character */
+      haystack = utf8codepoint(maybeMatch, &throwaway_codepoint);
+    }
+  }
+
+  /* no match */
+  return utf8_null;
+}
+
+utf8_constexpr14_impl utf8_int8_t *utf8casestr(const utf8_int8_t *haystack,
+                                               const utf8_int8_t *needle) {
+  /* if needle has no utf8 codepoints before the null terminating
+   * byte then return haystack */
+  if ('\0' == *needle) {
+    return (utf8_int8_t *)haystack;
+  }
+
+  for (;;) {
+    const utf8_int8_t *maybeMatch = haystack;
+    const utf8_int8_t *n = needle;
+    utf8_int32_t h_cp = 0, n_cp = 0;
+
+    /* Get the next code point and track it */
+    const utf8_int8_t *nextH = haystack = utf8codepoint(haystack, &h_cp);
+    n = utf8codepoint(n, &n_cp);
+
+    while ((0 != h_cp) && (0 != n_cp)) {
+      h_cp = utf8lwrcodepoint(h_cp);
+      n_cp = utf8lwrcodepoint(n_cp);
+
+      /* if we find a mismatch, bail out! */
+      if (h_cp != n_cp) {
+        break;
+      }
+
+      haystack = utf8codepoint(haystack, &h_cp);
+      n = utf8codepoint(n, &n_cp);
+    }
+
+    if (0 == n_cp) {
+      /* we found the whole utf8 string for needle in haystack at
+       * maybeMatch, so return it */
+      return (utf8_int8_t *)maybeMatch;
+    }
+
+    if (0 == h_cp) {
+      /* no match */
+      return utf8_null;
+    }
+
+    /* Roll back to the next code point in the haystack to test */
+    haystack = nextH;
+  }
+}
+
+utf8_constexpr14_impl utf8_int8_t *utf8valid(const utf8_int8_t *str) {
+  return utf8nvalid(str, SIZE_MAX);
+}
+
+utf8_constexpr14_impl utf8_int8_t *utf8nvalid(const utf8_int8_t *str,
+                                              size_t n) {
+  const utf8_int8_t *t = str;
+  size_t consumed = 0;
+
+  while ((void)(consumed = (size_t)(str - t)), consumed < n && '\0' != *str) {
+    const size_t remaining = n - consumed;
+
+    if (0xf0 == (0xf8 & *str)) {
+      /* ensure that there's 4 bytes or more remaining */
+      if (remaining < 4) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* ensure each of the 3 following bytes in this 4-byte
+       * utf8 codepoint began with 0b10xxxxxx */
+      if ((0x80 != (0xc0 & str[1])) || (0x80 != (0xc0 & str[2])) ||
+          (0x80 != (0xc0 & str[3]))) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* ensure that our utf8 codepoint ended after 4 bytes */
+      if ((remaining != 4) && (0x80 == (0xc0 & str[4]))) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* ensure that the top 5 bits of this 4-byte utf8
+       * codepoint were not 0, as then we could have used
+       * one of the smaller encodings */
+      if ((0 == (0x07 & str[0])) && (0 == (0x30 & str[1]))) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* 4-byte utf8 code point (began with 0b11110xxx) */
+      str += 4;
+    } else if (0xe0 == (0xf0 & *str)) {
+      /* ensure that there's 3 bytes or more remaining */
+      if (remaining < 3) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* ensure each of the 2 following bytes in this 3-byte
+       * utf8 codepoint began with 0b10xxxxxx */
+      if ((0x80 != (0xc0 & str[1])) || (0x80 != (0xc0 & str[2]))) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* ensure that our utf8 codepoint ended after 3 bytes */
+      if ((remaining != 3) && (0x80 == (0xc0 & str[3]))) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* ensure that the top 5 bits of this 3-byte utf8
+       * codepoint were not 0, as then we could have used
+       * one of the smaller encodings */
+      if ((0 == (0x0f & str[0])) && (0 == (0x20 & str[1]))) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* 3-byte utf8 code point (began with 0b1110xxxx) */
+      str += 3;
+    } else if (0xc0 == (0xe0 & *str)) {
+      /* ensure that there's 2 bytes or more remaining */
+      if (remaining < 2) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* ensure the 1 following byte in this 2-byte
+       * utf8 codepoint began with 0b10xxxxxx */
+      if (0x80 != (0xc0 & str[1])) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* ensure that our utf8 codepoint ended after 2 bytes */
+      if ((remaining != 2) && (0x80 == (0xc0 & str[2]))) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* ensure that the top 4 bits of this 2-byte utf8
+       * codepoint were not 0, as then we could have used
+       * one of the smaller encodings */
+      if (0 == (0x1e & str[0])) {
+        return (utf8_int8_t *)str;
+      }
+
+      /* 2-byte utf8 code point (began with 0b110xxxxx) */
+      str += 2;
+    } else if (0x00 == (0x80 & *str)) {
+      /* 1-byte ascii (began with 0b0xxxxxxx) */
+      str += 1;
+    } else {
+      /* we have an invalid 0b1xxxxxxx utf8 code point entry */
+      return (utf8_int8_t *)str;
+    }
+  }
+
+  return utf8_null;
+}
+
+int utf8makevalid(utf8_int8_t *str, const utf8_int32_t replacement) {
+  utf8_int8_t *read = str;
+  utf8_int8_t *write = read;
+  const utf8_int8_t r = (utf8_int8_t)replacement;
+  utf8_int32_t codepoint = 0;
+
+  if (replacement > 0x7f) {
+    return -1;
+  }
+
+  while ('\0' != *read) {
+    if (0xf0 == (0xf8 & *read)) {
+      /* ensure each of the 3 following bytes in this 4-byte
+       * utf8 codepoint began with 0b10xxxxxx */
+      if ((0x80 != (0xc0 & read[1])) || (0x80 != (0xc0 & read[2])) ||
+          (0x80 != (0xc0 & read[3]))) {
+        *write++ = r;
+        read++;
+        continue;
+      }
+
+      /* 4-byte utf8 code point (began with 0b11110xxx) */
+      read = utf8codepoint(read, &codepoint);
+      write = utf8catcodepoint(write, codepoint, 4);
+    } else if (0xe0 == (0xf0 & *read)) {
+      /* ensure each of the 2 following bytes in this 3-byte
+       * utf8 codepoint began with 0b10xxxxxx */
+      if ((0x80 != (0xc0 & read[1])) || (0x80 != (0xc0 & read[2]))) {
+        *write++ = r;
+        read++;
+        continue;
+      }
+
+      /* 3-byte utf8 code point (began with 0b1110xxxx) */
+      read = utf8codepoint(read, &codepoint);
+      write = utf8catcodepoint(write, codepoint, 3);
+    } else if (0xc0 == (0xe0 & *read)) {
+      /* ensure the 1 following byte in this 2-byte
+       * utf8 codepoint began with 0b10xxxxxx */
+      if (0x80 != (0xc0 & read[1])) {
+        *write++ = r;
+        read++;
+        continue;
+      }
+
+      /* 2-byte utf8 code point (began with 0b110xxxxx) */
+      read = utf8codepoint(read, &codepoint);
+      write = utf8catcodepoint(write, codepoint, 2);
+    } else if (0x00 == (0x80 & *read)) {
+      /* 1-byte ascii (began with 0b0xxxxxxx) */
+      read = utf8codepoint(read, &codepoint);
+      write = utf8catcodepoint(write, codepoint, 1);
+    } else {
+      /* if we got here then we've got a dangling continuation (0b10xxxxxx) */
+      *write++ = r;
+      read++;
+      continue;
+    }
+  }
+
+  *write = '\0';
+
+  return 0;
+}
+
+utf8_constexpr14_impl utf8_int8_t *
+utf8codepoint(const utf8_int8_t *utf8_restrict str,
+              utf8_int32_t *utf8_restrict out_codepoint) {
+  if (0xf0 == (0xf8 & str[0])) {
+    /* 4 byte utf8 codepoint */
+    *out_codepoint = ((0x07 & str[0]) << 18) | ((0x3f & str[1]) << 12) |
+                     ((0x3f & str[2]) << 6) | (0x3f & str[3]);
+    str += 4;
+  } else if (0xe0 == (0xf0 & str[0])) {
+    /* 3 byte utf8 codepoint */
+    *out_codepoint =
+        ((0x0f & str[0]) << 12) | ((0x3f & str[1]) << 6) | (0x3f & str[2]);
+    str += 3;
+  } else if (0xc0 == (0xe0 & str[0])) {
+    /* 2 byte utf8 codepoint */
+    *out_codepoint = ((0x1f & str[0]) << 6) | (0x3f & str[1]);
+    str += 2;
+  } else {
+    /* 1 byte utf8 codepoint otherwise */
+    *out_codepoint = str[0];
+    str += 1;
+  }
+
+  return (utf8_int8_t *)str;
+}
+
+utf8_constexpr14_impl size_t utf8codepointcalcsize(const utf8_int8_t *str) {
+  if (0xf0 == (0xf8 & str[0])) {
+    /* 4 byte utf8 codepoint */
+    return 4;
+  } else if (0xe0 == (0xf0 & str[0])) {
+    /* 3 byte utf8 codepoint */
+    return 3;
+  } else if (0xc0 == (0xe0 & str[0])) {
+    /* 2 byte utf8 codepoint */
+    return 2;
+  }
+
+  /* 1 byte utf8 codepoint otherwise */
+  return 1;
+}
+
+utf8_constexpr14_impl size_t utf8codepointsize(utf8_int32_t chr) {
+  if (0 == ((utf8_int32_t)0xffffff80 & chr)) {
+    return 1;
+  } else if (0 == ((utf8_int32_t)0xfffff800 & chr)) {
+    return 2;
+  } else if (0 == ((utf8_int32_t)0xffff0000 & chr)) {
+    return 3;
+  } else { /* if (0 == ((int)0xffe00000 & chr)) { */
+    return 4;
+  }
+}
+
+utf8_int8_t *utf8catcodepoint(utf8_int8_t *str, utf8_int32_t chr, size_t n) {
+  if (0 == ((utf8_int32_t)0xffffff80 & chr)) {
+    /* 1-byte/7-bit ascii
+     * (0b0xxxxxxx) */
+    if (n < 1) {
+      return utf8_null;
+    }
+    str[0] = (utf8_int8_t)chr;
+    str += 1;
+  } else if (0 == ((utf8_int32_t)0xfffff800 & chr)) {
+    /* 2-byte/11-bit utf8 code point
+     * (0b110xxxxx 0b10xxxxxx) */
+    if (n < 2) {
+      return utf8_null;
+    }
+    str[0] = (utf8_int8_t)(0xc0 | (utf8_int8_t)((chr >> 6) & 0x1f));
+    str[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
+    str += 2;
+  } else if (0 == ((utf8_int32_t)0xffff0000 & chr)) {
+    /* 3-byte/16-bit utf8 code point
+     * (0b1110xxxx 0b10xxxxxx 0b10xxxxxx) */
+    if (n < 3) {
+      return utf8_null;
+    }
+    str[0] = (utf8_int8_t)(0xe0 | (utf8_int8_t)((chr >> 12) & 0x0f));
+    str[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
+    str[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
+    str += 3;
+  } else { /* if (0 == ((int)0xffe00000 & chr)) { */
+    /* 4-byte/21-bit utf8 code point
+     * (0b11110xxx 0b10xxxxxx 0b10xxxxxx 0b10xxxxxx) */
+    if (n < 4) {
+      return utf8_null;
+    }
+    str[0] = (utf8_int8_t)(0xf0 | (utf8_int8_t)((chr >> 18) & 0x07));
+    str[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 12) & 0x3f));
+    str[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
+    str[3] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
+    str += 4;
+  }
+
+  return str;
+}
+
+utf8_constexpr14_impl int utf8islower(utf8_int32_t chr) {
+  return chr != utf8uprcodepoint(chr);
+}
+
+utf8_constexpr14_impl int utf8isupper(utf8_int32_t chr) {
+  return chr != utf8lwrcodepoint(chr);
+}
+
+void utf8lwr(utf8_int8_t *utf8_restrict str) {
+  utf8_int32_t cp = 0;
+  utf8_int8_t *pn = utf8codepoint(str, &cp);
+
+  while (cp != 0) {
+    const utf8_int32_t lwr_cp = utf8lwrcodepoint(cp);
+    const size_t size = utf8codepointsize(lwr_cp);
+
+    if (lwr_cp != cp) {
+      utf8catcodepoint(str, lwr_cp, size);
+    }
+
+    str = pn;
+    pn = utf8codepoint(str, &cp);
+  }
+}
+
+void utf8upr(utf8_int8_t *utf8_restrict str) {
+  utf8_int32_t cp = 0;
+  utf8_int8_t *pn = utf8codepoint(str, &cp);
+
+  while (cp != 0) {
+    const utf8_int32_t lwr_cp = utf8uprcodepoint(cp);
+    const size_t size = utf8codepointsize(lwr_cp);
+
+    if (lwr_cp != cp) {
+      utf8catcodepoint(str, lwr_cp, size);
+    }
+
+    str = pn;
+    pn = utf8codepoint(str, &cp);
+  }
+}
+
+utf8_constexpr14_impl utf8_int32_t utf8lwrcodepoint(utf8_int32_t cp) {
+  if (((0x0041 <= cp) && (0x005a >= cp)) ||
+      ((0x00c0 <= cp) && (0x00d6 >= cp)) ||
+      ((0x00d8 <= cp) && (0x00de >= cp)) ||
+      ((0x0391 <= cp) && (0x03a1 >= cp)) ||
+      ((0x03a3 <= cp) && (0x03ab >= cp)) ||
+      ((0x0410 <= cp) && (0x042f >= cp))) {
+    cp += 32;
+  } else if ((0x0400 <= cp) && (0x040f >= cp)) {
+    cp += 80;
+  } else if (((0x0100 <= cp) && (0x012f >= cp)) ||
+             ((0x0132 <= cp) && (0x0137 >= cp)) ||
+             ((0x014a <= cp) && (0x0177 >= cp)) ||
+             ((0x0182 <= cp) && (0x0185 >= cp)) ||
+             ((0x01a0 <= cp) && (0x01a5 >= cp)) ||
+             ((0x01de <= cp) && (0x01ef >= cp)) ||
+             ((0x01f8 <= cp) && (0x021f >= cp)) ||
+             ((0x0222 <= cp) && (0x0233 >= cp)) ||
+             ((0x0246 <= cp) && (0x024f >= cp)) ||
+             ((0x03d8 <= cp) && (0x03ef >= cp)) ||
+             ((0x0460 <= cp) && (0x0481 >= cp)) ||
+             ((0x048a <= cp) && (0x04ff >= cp))) {
+    cp |= 0x1;
+  } else if (((0x0139 <= cp) && (0x0148 >= cp)) ||
+             ((0x0179 <= cp) && (0x017e >= cp)) ||
+             ((0x01af <= cp) && (0x01b0 >= cp)) ||
+             ((0x01b3 <= cp) && (0x01b6 >= cp)) ||
+             ((0x01cd <= cp) && (0x01dc >= cp))) {
+    cp += 1;
+    cp &= ~0x1;
+  } else {
+    switch (cp) {
+    default:
+      break;
+    case 0x0178:
+      cp = 0x00ff;
+      break;
+    case 0x0243:
+      cp = 0x0180;
+      break;
+    case 0x018e:
+      cp = 0x01dd;
+      break;
+    case 0x023d:
+      cp = 0x019a;
+      break;
+    case 0x0220:
+      cp = 0x019e;
+      break;
+    case 0x01b7:
+      cp = 0x0292;
+      break;
+    case 0x01c4:
+      cp = 0x01c6;
+      break;
+    case 0x01c7:
+      cp = 0x01c9;
+      break;
+    case 0x01ca:
+      cp = 0x01cc;
+      break;
+    case 0x01f1:
+      cp = 0x01f3;
+      break;
+    case 0x01f7:
+      cp = 0x01bf;
+      break;
+    case 0x0187:
+      cp = 0x0188;
+      break;
+    case 0x018b:
+      cp = 0x018c;
+      break;
+    case 0x0191:
+      cp = 0x0192;
+      break;
+    case 0x0198:
+      cp = 0x0199;
+      break;
+    case 0x01a7:
+      cp = 0x01a8;
+      break;
+    case 0x01ac:
+      cp = 0x01ad;
+      break;
+    case 0x01b8:
+      cp = 0x01b9;
+      break;
+    case 0x01bc:
+      cp = 0x01bd;
+      break;
+    case 0x01f4:
+      cp = 0x01f5;
+      break;
+    case 0x023b:
+      cp = 0x023c;
+      break;
+    case 0x0241:
+      cp = 0x0242;
+      break;
+    case 0x03fd:
+      cp = 0x037b;
+      break;
+    case 0x03fe:
+      cp = 0x037c;
+      break;
+    case 0x03ff:
+      cp = 0x037d;
+      break;
+    case 0x037f:
+      cp = 0x03f3;
+      break;
+    case 0x0386:
+      cp = 0x03ac;
+      break;
+    case 0x0388:
+      cp = 0x03ad;
+      break;
+    case 0x0389:
+      cp = 0x03ae;
+      break;
+    case 0x038a:
+      cp = 0x03af;
+      break;
+    case 0x038c:
+      cp = 0x03cc;
+      break;
+    case 0x038e:
+      cp = 0x03cd;
+      break;
+    case 0x038f:
+      cp = 0x03ce;
+      break;
+    case 0x0370:
+      cp = 0x0371;
+      break;
+    case 0x0372:
+      cp = 0x0373;
+      break;
+    case 0x0376:
+      cp = 0x0377;
+      break;
+    case 0x03f4:
+      cp = 0x03b8;
+      break;
+    case 0x03cf:
+      cp = 0x03d7;
+      break;
+    case 0x03f9:
+      cp = 0x03f2;
+      break;
+    case 0x03f7:
+      cp = 0x03f8;
+      break;
+    case 0x03fa:
+      cp = 0x03fb;
+      break;
+    }
+  }
+
+  return cp;
+}
+
+utf8_constexpr14_impl utf8_int32_t utf8uprcodepoint(utf8_int32_t cp) {
+  if (((0x0061 <= cp) && (0x007a >= cp)) ||
+      ((0x00e0 <= cp) && (0x00f6 >= cp)) ||
+      ((0x00f8 <= cp) && (0x00fe >= cp)) ||
+      ((0x03b1 <= cp) && (0x03c1 >= cp)) ||
+      ((0x03c3 <= cp) && (0x03cb >= cp)) ||
+      ((0x0430 <= cp) && (0x044f >= cp))) {
+    cp -= 32;
+  } else if ((0x0450 <= cp) && (0x045f >= cp)) {
+    cp -= 80;
+  } else if (((0x0100 <= cp) && (0x012f >= cp)) ||
+             ((0x0132 <= cp) && (0x0137 >= cp)) ||
+             ((0x014a <= cp) && (0x0177 >= cp)) ||
+             ((0x0182 <= cp) && (0x0185 >= cp)) ||
+             ((0x01a0 <= cp) && (0x01a5 >= cp)) ||
+             ((0x01de <= cp) && (0x01ef >= cp)) ||
+             ((0x01f8 <= cp) && (0x021f >= cp)) ||
+             ((0x0222 <= cp) && (0x0233 >= cp)) ||
+             ((0x0246 <= cp) && (0x024f >= cp)) ||
+             ((0x03d8 <= cp) && (0x03ef >= cp)) ||
+             ((0x0460 <= cp) && (0x0481 >= cp)) ||
+             ((0x048a <= cp) && (0x04ff >= cp))) {
+    cp &= ~0x1;
+  } else if (((0x0139 <= cp) && (0x0148 >= cp)) ||
+             ((0x0179 <= cp) && (0x017e >= cp)) ||
+             ((0x01af <= cp) && (0x01b0 >= cp)) ||
+             ((0x01b3 <= cp) && (0x01b6 >= cp)) ||
+             ((0x01cd <= cp) && (0x01dc >= cp))) {
+    cp -= 1;
+    cp |= 0x1;
+  } else {
+    switch (cp) {
+    default:
+      break;
+    case 0x00ff:
+      cp = 0x0178;
+      break;
+    case 0x0180:
+      cp = 0x0243;
+      break;
+    case 0x01dd:
+      cp = 0x018e;
+      break;
+    case 0x019a:
+      cp = 0x023d;
+      break;
+    case 0x019e:
+      cp = 0x0220;
+      break;
+    case 0x0292:
+      cp = 0x01b7;
+      break;
+    case 0x01c6:
+      cp = 0x01c4;
+      break;
+    case 0x01c9:
+      cp = 0x01c7;
+      break;
+    case 0x01cc:
+      cp = 0x01ca;
+      break;
+    case 0x01f3:
+      cp = 0x01f1;
+      break;
+    case 0x01bf:
+      cp = 0x01f7;
+      break;
+    case 0x0188:
+      cp = 0x0187;
+      break;
+    case 0x018c:
+      cp = 0x018b;
+      break;
+    case 0x0192:
+      cp = 0x0191;
+      break;
+    case 0x0199:
+      cp = 0x0198;
+      break;
+    case 0x01a8:
+      cp = 0x01a7;
+      break;
+    case 0x01ad:
+      cp = 0x01ac;
+      break;
+    case 0x01b9:
+      cp = 0x01b8;
+      break;
+    case 0x01bd:
+      cp = 0x01bc;
+      break;
+    case 0x01f5:
+      cp = 0x01f4;
+      break;
+    case 0x023c:
+      cp = 0x023b;
+      break;
+    case 0x0242:
+      cp = 0x0241;
+      break;
+    case 0x037b:
+      cp = 0x03fd;
+      break;
+    case 0x037c:
+      cp = 0x03fe;
+      break;
+    case 0x037d:
+      cp = 0x03ff;
+      break;
+    case 0x03f3:
+      cp = 0x037f;
+      break;
+    case 0x03ac:
+      cp = 0x0386;
+      break;
+    case 0x03ad:
+      cp = 0x0388;
+      break;
+    case 0x03ae:
+      cp = 0x0389;
+      break;
+    case 0x03af:
+      cp = 0x038a;
+      break;
+    case 0x03cc:
+      cp = 0x038c;
+      break;
+    case 0x03cd:
+      cp = 0x038e;
+      break;
+    case 0x03ce:
+      cp = 0x038f;
+      break;
+    case 0x0371:
+      cp = 0x0370;
+      break;
+    case 0x0373:
+      cp = 0x0372;
+      break;
+    case 0x0377:
+      cp = 0x0376;
+      break;
+    case 0x03d1:
+      cp = 0x0398;
+      break;
+    case 0x03d7:
+      cp = 0x03cf;
+      break;
+    case 0x03f2:
+      cp = 0x03f9;
+      break;
+    case 0x03f8:
+      cp = 0x03f7;
+      break;
+    case 0x03fb:
+      cp = 0x03fa;
+      break;
+    }
+  }
+
+  return cp;
+}
+
+utf8_constexpr14_impl utf8_int8_t *
+utf8rcodepoint(const utf8_int8_t *utf8_restrict str,
+               utf8_int32_t *utf8_restrict out_codepoint) {
+  const utf8_int8_t *s = (const utf8_int8_t *)str;
+
+  if (0xf0 == (0xf8 & s[0])) {
+    /* 4 byte utf8 codepoint */
+    *out_codepoint = ((0x07 & s[0]) << 18) | ((0x3f & s[1]) << 12) |
+                     ((0x3f & s[2]) << 6) | (0x3f & s[3]);
+  } else if (0xe0 == (0xf0 & s[0])) {
+    /* 3 byte utf8 codepoint */
+    *out_codepoint =
+        ((0x0f & s[0]) << 12) | ((0x3f & s[1]) << 6) | (0x3f & s[2]);
+  } else if (0xc0 == (0xe0 & s[0])) {
+    /* 2 byte utf8 codepoint */
+    *out_codepoint = ((0x1f & s[0]) << 6) | (0x3f & s[1]);
+  } else {
+    /* 1 byte utf8 codepoint otherwise */
+    *out_codepoint = s[0];
+  }
+
+  do {
+    s--;
+  } while ((0 != (0x80 & s[0])) && (0x80 == (0xc0 & s[0])));
+
+  return (utf8_int8_t *)s;
+}
+
+#undef utf8_restrict
+#undef utf8_constexpr14
+#undef utf8_null
+
+#ifdef utf8_cplusplus
+} /* extern "C" */
+#endif
+
+#if defined(__clang__)
+#pragma clang diagnostic pop
+#endif
+
+#endif /* SHEREDOM_UTF8_H_INCLUDED */
diff --git a/tests/datetime/import.du b/tests/datetime/import.du
index 042d113be..bfcbafa9e 100644
--- a/tests/datetime/import.du
+++ b/tests/datetime/import.du
@@ -6,4 +6,4 @@
 
 import "constants.du";
 import "strftime.du";
-import "strptime.du";
+//import "strptime.du";
diff --git a/tests/strings/find.du b/tests/strings/find.du
index 2ba1def7b..59ec6af2c 100644
--- a/tests/strings/find.du
+++ b/tests/strings/find.du
@@ -21,7 +21,12 @@ class TestStringFind < UnitTest {
     }
 
     testStringFindThird() {
-        this.assertEquals("Dictu is great! Dictu is great! Dictu is great!".find("Dictu", 3), 37);
+        this.assertEquals("Dictu is great! Dictu is great! Dictu is great!".find("Dictu", 3), 32);
+    }
+    testStringFindUnicode() {
+        this.assertEquals("💻😃😃😃a".find("💻"), 0);
+        this.assertEquals("💻😃😃😃a".find("😃",2), 2);
+        this.assertEquals("💻😃😃😃a".find("💻",2), -1);
     }
 
     testStringFindDoesntExist() {
diff --git a/tests/strings/isLower.du b/tests/strings/isLower.du
index 5b94fd345..324d7ce6e 100644
--- a/tests/strings/isLower.du
+++ b/tests/strings/isLower.du
@@ -16,6 +16,13 @@ class TestStringIsLower < UnitTest {
         this.assertFalsey("D".isLower());
         this.assertFalsey("Maple".isLower());
     }
+    testStringIsLowerUnicode() {
+        this.assertTruthy("dä".isLower());
+        this.assertTruthy("dögog".isLower());
+        this.assertTruthy("g0ä0d!".isLower());
+        this.assertFalsey("Ä".isLower());
+        this.assertFalsey("Ma😁ple".isLower());
+    }
 }
 
 TestStringIsLower().run();
diff --git a/tests/strings/isUpper.du b/tests/strings/isUpper.du
index f7d3294cb..c6444473f 100644
--- a/tests/strings/isUpper.du
+++ b/tests/strings/isUpper.du
@@ -15,6 +15,12 @@ class TestStringIsUpper < UnitTest {
         this.assertTruthy("G00D!".isUpper());
         this.assertFalsey("Maple".isUpper());
     }
+    testStringIsUpperUnicode() {
+        this.assertTruthy("😆D".isUpper());
+        this.assertTruthy("DO😆G".isUpper());
+        this.assertTruthy("G😆00D!".isUpper());
+        this.assertFalsey("Ma😆ple".isUpper());
+    }
 }
 
 TestStringIsUpper().run();
diff --git a/tests/strings/len.du b/tests/strings/len.du
index c8fdca006..73f0a864f 100644
--- a/tests/strings/len.du
+++ b/tests/strings/len.du
@@ -11,9 +11,12 @@ class TestStringLen < UnitTest {
     testStringLen() {
         this.assertType("qwert".len(), "number");
         this.assertEquals("qwert".len(), 5);
-        this.assertEquals("ασδφγ".len(), 10);
+        this.assertEquals("ασδφγ".len(), 5);
         this.assertEquals(('test' + 'ing').len(), 7);
     }
+    testStringLenUnicode() {
+        this.assertEquals("🤪".len(), 1);
+    }
 }
 
 TestStringLen().run();
\ No newline at end of file
diff --git a/tests/strings/split.du b/tests/strings/split.du
index 165e2d76e..def59114e 100644
--- a/tests/strings/split.du
+++ b/tests/strings/split.du
@@ -28,6 +28,12 @@ class TestStringSplit < UnitTest {
         this.assertEquals("Dictu is great!".split(" ", 3), ["Dictu", "is", "great!"]);
         this.assertEquals("Dictu is great!".split(" ", 4), ["Dictu", "is", "great!"]);
     }
+
+    testStringSplitUnicode() {
+
+        this.assertEquals("Dictu💻is💻very💻😃Cool😃".split("💻"), ["Dictu", "is", "very", "😃Cool😃"]);
+        this.assertEquals("Dictu is very😃Cool😃!".split("😃Cool😃"), ["Dictu is very", "!"]);
+    }
     
     testStringSplitOptionalSinglular() {
         this.assertEquals("Dictu is great!".split("", -2), ["D", "i", "c", "t", "u", " ", "i", "s", " ", "g", "r", "e", "a", "t", "!"]);
diff --git a/tests/strings/startsWith.du b/tests/strings/startsWith.du
index c5b8272d1..c145034a6 100644
--- a/tests/strings/startsWith.du
+++ b/tests/strings/startsWith.du
@@ -19,6 +19,10 @@ class TestStringStartsWith < UnitTest {
         this.assertFalsey("test".startsWith("!@£$%"));
         this.assertFalsey("test".startsWith("123456789"));
     }
+    testStringStartsWithUnicode() {
+        this.assertTruthy("💻test".startsWith("💻t"));
+        this.assertFalsey("💻test".startsWith("💻te💻"));
+    }
 }
 
 TestStringStartsWith().run();
\ No newline at end of file
diff --git a/tests/uuid/generate.du b/tests/uuid/generate.du
index b2297b9e8..6dcc64458 100644
--- a/tests/uuid/generate.du
+++ b/tests/uuid/generate.du
@@ -16,7 +16,7 @@ class TestUUIDGenerate < UnitTest {
         this.assertTruthy(ret.success());
         const uuid = ret.unwrap();
         this.assertType(type(uuid), "string");
-        this.assertTruthy(uuid.len() == 37);
+        this.assertTruthy(uuid.byteLen() == 37);
         print(uuid);
     }
 }
diff --git a/tests/uuid/generateRandom.du b/tests/uuid/generateRandom.du
index 0a66edcb8..d3e6afcaf 100644
--- a/tests/uuid/generateRandom.du
+++ b/tests/uuid/generateRandom.du
@@ -16,7 +16,7 @@ class TestUUIDGenerateRandom < UnitTest {
         this.assertTruthy(ret.success());
         const uuid = ret.unwrap();
         this.assertType(type(uuid), "string");
-        this.assertTruthy(uuid.len() == 37);
+        this.assertTruthy(uuid.byteLen() == 37);
     }
 }
 
diff --git a/tests/uuid/generateTime.du b/tests/uuid/generateTime.du
index efe051669..5cfa84b10 100644
--- a/tests/uuid/generateTime.du
+++ b/tests/uuid/generateTime.du
@@ -16,7 +16,7 @@ class TestUUIDGenerateTime < UnitTest {
         this.assertTruthy(ret.success());
         const uuid = ret.unwrap();
         this.assertType(type(uuid), "string");
-        this.assertTruthy(uuid.len() == 37);
+        this.assertTruthy(uuid.byteLen() == 37);
     }
 }
 

From 28d47a7ae1676d110b4278be30693ffc457b7275 Mon Sep 17 00:00:00 2001
From: "Liz3 (Yann HN)" <accs@liz3.net>
Date: Wed, 18 Sep 2024 03:19:51 +0200
Subject: [PATCH 02/31] fix compile error

---
 src/vm/datatypes/strings.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index fcb05bf80..794e68036 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -196,7 +196,7 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
             count++;
             token = utf8str(tmp, delimiter);
 
-            Value str = OBJ_VAL(copyString(vm, tmp, token == NULL ? utf8size_lazy(tmp) : (token-tmp)));
+            Value str = OBJ_VAL(copyString(vm, tmp, token == NULL ? utf8size_lazy(tmp) : ((size_t)(token-tmp))));
 
             // Push to stack to avoid GC
             push(vm, str);

From 9b3f32a903c44d7574a879783516acd4c7746162 Mon Sep 17 00:00:00 2001
From: "Liz3 (Yann HN)" <accs@liz3.net>
Date: Wed, 18 Sep 2024 03:24:09 +0200
Subject: [PATCH 03/31] validity check

---
 src/vm/object.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/vm/object.c b/src/vm/object.c
index 587b521c0..2b35b4844 100644
--- a/src/vm/object.c
+++ b/src/vm/object.c
@@ -153,7 +153,10 @@ static ObjString *allocateString(DictuVM *vm, char *chars, int length,
     string->length = length;
     string->chars = chars;
     string->hash = hash;
-    string->character_len = utf8len(chars);
+    if(utf8valid(chars) == 0)
+        string->character_len = utf8len(chars);
+    else
+        string->character_len = length;
     push(vm, OBJ_VAL(string));
     tableSet(vm, &vm->strings, string, NIL_VAL);
     pop(vm);

From 57d601ab979068e26857eb18e0813c44a59dba09 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Sat, 26 Oct 2024 23:02:37 +0200
Subject: [PATCH 04/31] fix: need this fix for it to compile

---
 src/optionals/sqlite/sqlite3.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/optionals/sqlite/sqlite3.c b/src/optionals/sqlite/sqlite3.c
index 893718801..0955c8544 100644
--- a/src/optionals/sqlite/sqlite3.c
+++ b/src/optionals/sqlite/sqlite3.c
@@ -56382,7 +56382,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen(
   const char *zUri = 0;    /* URI args to copy */
   int nUriByte = 1;        /* Number of bytes of URI args at *zUri */
   int nUri = 0;            /* Number of URI parameters */
-
+  (void)nUri;
   /* Figure out how much space is required for each journal file-handle
   ** (there are two of them, the main journal and the sub-journal).  */
   journalFileSize = ROUND8(sqlite3JournalSize(pVfs));

From 67ad5fe40271cf4b2fd2ed76ed0a0af77fbd46fe Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Sun, 27 Oct 2024 00:30:23 +0200
Subject: [PATCH 05/31] implement general utf-8 check, implement the indexing,
 slicing

---
 src/vm/datatypes/strings.c | 105 +++++++++++++++++++++++++++++++++++--
 src/vm/object.c            |  42 ++++++++++++---
 src/vm/object.h            |   2 +
 src/vm/vm.c                |  45 ++++++++++++----
 4 files changed, 176 insertions(+), 18 deletions(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 794e68036..4f4bac699 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -15,6 +15,10 @@ static Value lenString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
+    if(string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8, consider using byteLen()");
+        return EMPTY_VAL;
+    }
     return NUMBER_VAL(string->character_len);
 }
 
@@ -65,6 +69,12 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "format() takes at least 1 argument (%d given)", argCount);
         return EMPTY_VAL;
     }
+    ObjString *string = AS_STRING(args[0]);
+
+    if(string->character_len == -1) {
+     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     return EMPTY_VAL;
+    }
 
     int length = 0;
     char **replaceStrings = ALLOCATE(vm, char*, argCount);
@@ -83,7 +93,6 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
         length += utf8size_lazy(replaceStrings[j - 1]);
     }
 
-    ObjString *string = AS_STRING(args[0]);
 
     int stringLen = string->length + 1;
     char *tmp = string->chars;
@@ -145,6 +154,11 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
+    if(string->character_len == -1) {
+     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     return EMPTY_VAL;
+     }
+
     char *delimiter = AS_CSTRING(args[1]);
     int maxSplit = string->length + 1;
 
@@ -226,11 +240,18 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
 }
 
 static Value containsString(DictuVM *vm, int argCount, Value *args) {
+
     if (argCount != 1) {
         runtimeError(vm, "contains() takes 1 argument (%d given)", argCount);
         return EMPTY_VAL;
     }
-
+    {
+        ObjString *string = AS_STRING(args[0]);
+        if(string->character_len == -1) {
+           runtimeError(vm, "String contains invalid UTF-8", argCount);
+           return EMPTY_VAL;
+       }
+    }
     if (!IS_STRING(args[1])) {
         runtimeError(vm, "Argument passed to contains() must be a string");
         return EMPTY_VAL;
@@ -251,7 +272,13 @@ static Value findString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "find() takes either 1 or 2 arguments (%d given)", argCount);
         return EMPTY_VAL;
     }
-
+    {
+        ObjString *string = AS_STRING(args[0]);
+        if(string->character_len == -1) {
+           runtimeError(vm, "String contains invalid UTF-8", argCount);
+           return EMPTY_VAL;
+       }
+    }
     int index = 1;
 
     if (argCount == 2) {
@@ -294,6 +321,13 @@ static Value findLastString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "findLast() takes 1 argument (%d given)", argCount);
         return EMPTY_VAL;
     }
+    {
+        ObjString *string = AS_STRING(args[0]);
+        if(string->character_len == -1) {
+           runtimeError(vm, "String contains invalid UTF-8", argCount);
+           return EMPTY_VAL;
+       }
+    }
 
     const char *str = AS_CSTRING(args[0]);
     const char *ss = AS_CSTRING(args[1]);
@@ -342,6 +376,13 @@ static Value replaceString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "Arguments passed to replace() must be a strings");
         return EMPTY_VAL;
     }
+    {
+        ObjString *string = AS_STRING(args[0]);
+        if(string->character_len == -1) {
+           runtimeError(vm, "String contains invalid UTF-8", argCount);
+           return EMPTY_VAL;
+       }
+    }
 
     // Pop values off the stack
     Value stringValue = args[0];
@@ -399,6 +440,10 @@ static Value lowerString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
+    if(string->character_len == -1) {
+     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     return EMPTY_VAL;
+    }
     char *temp = ALLOCATE(vm, char, string->length + 1);
     memcpy(temp, string->chars, string->length);
     utf8lwr(temp);
@@ -414,6 +459,11 @@ static Value upperString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
+    if(string->character_len == -1) {
+     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     return EMPTY_VAL;
+    }
+
     char *temp = ALLOCATE(vm, char, string->length + 1);
     memcpy(temp, string->chars, string->length);
     temp[string->length] = 0x00;
@@ -434,6 +484,11 @@ static Value startsWithString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
+    if(string->character_len == -1) {
+     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     return EMPTY_VAL;
+    }
+
     ObjString *start = AS_STRING(args[1]);
     if(string->length < start->length)
         return BOOL_VAL(false);
@@ -453,6 +508,11 @@ static Value endsWithString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
+    if(string->character_len == -1) {
+     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     return EMPTY_VAL;
+    }
+
     ObjString *suffix = AS_STRING(args[1]);
 
     if (string->length < suffix->length) {
@@ -537,6 +597,13 @@ static Value countString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "Argument passed to count() must be a string");
         return EMPTY_VAL;
     }
+    {
+        ObjString *string = AS_STRING(args[0]);
+        if(string->character_len == -1) {
+           runtimeError(vm, "String contains invalid UTF-8", argCount);
+           return EMPTY_VAL;
+       }
+    }
 
     char *haystack = AS_CSTRING(args[0]);
     char *needle = AS_CSTRING(args[1]);
@@ -558,6 +625,10 @@ static Value wordCountString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     
+    if(string->character_len == -1) {
+     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     return EMPTY_VAL;
+    }
     int count = 0;
     int len = string->character_len;
     bool in = false;
@@ -586,6 +657,11 @@ static Value titleString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
+    if(string->character_len == -1) {
+     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     return EMPTY_VAL;
+    }
+
     char *temp = ALLOCATE(vm, char, string->length + 1);
     memcpy(temp, string->chars, string->length);
     char* ptr = temp;
@@ -643,6 +719,14 @@ static Value isUpperString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "isUpper() takes no arguments (%d given)", argCount);
         return EMPTY_VAL;
     }
+    {
+        ObjString *string = AS_STRING(args[0]);
+        if(string->character_len == -1) {
+           runtimeError(vm, "String contains invalid UTF-8", argCount);
+           return EMPTY_VAL;
+       }
+    }
+
 
     char *string = AS_CSTRING(args[0]);
     int len = utf8len(string);
@@ -667,6 +751,13 @@ static Value isLowerString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "isLower() takes no arguments (%d given)", argCount);
         return EMPTY_VAL;
     }
+    {
+        ObjString *string = AS_STRING(args[0]);
+        if(string->character_len == -1) {
+           runtimeError(vm, "String contains invalid UTF-8", argCount);
+           return EMPTY_VAL;
+       }
+    }
 
     char *string = AS_CSTRING(args[0]);
     int len = utf8len(string);
@@ -694,6 +785,10 @@ static Value collapseSpacesString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
+    if(string->character_len == -1) {
+       runtimeError(vm, "String contains invalid UTF-8", argCount);
+       return EMPTY_VAL;
+   }
     char *dest = ALLOCATE(vm, char, string->length + 1);
     char* ptr = string->chars;
     int targetLen = 0;
@@ -727,6 +822,10 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
+    if(string->character_len == -1) {
+       runtimeError(vm, "String contains invalid UTF-8", argCount);
+       return EMPTY_VAL;
+   }
     char *temp = ALLOCATE(vm, char, string->length + 1);
     memcpy(temp, string->chars, string->length);
     int len = AS_NUMBER(args[1]);
diff --git a/src/vm/object.c b/src/vm/object.c
index 2b35b4844..04cf28524 100644
--- a/src/vm/object.c
+++ b/src/vm/object.c
@@ -147,22 +147,26 @@ ObjNative *newNative(DictuVM *vm, NativeFn function) {
     return native;
 }
 
-static ObjString *allocateString(DictuVM *vm, char *chars, int length,
-                                 uint32_t hash) {
+static ObjString *allocateStringWithLen(DictuVM *vm, char *chars, int length,
+                                 uint32_t hash, int character_len) {
     ObjString *string = ALLOCATE_OBJ(vm, ObjString, OBJ_STRING);
     string->length = length;
     string->chars = chars;
     string->hash = hash;
-    if(utf8valid(chars) == 0)
-        string->character_len = utf8len(chars);
-    else
-        string->character_len = length;
+    string->character_len = character_len;
+
     push(vm, OBJ_VAL(string));
     tableSet(vm, &vm->strings, string, NIL_VAL);
     pop(vm);
     return string;
 }
 
+static ObjString *allocateString(DictuVM *vm, char *chars, int length,
+                                 uint32_t hash) {
+    int character_len = utf8valid(chars) == 0 ? utf8len(chars) : -1;
+    return allocateStringWithLen(vm, chars, length, hash, character_len);
+}
+
 ObjList *newList(DictuVM *vm) {
     ObjList *list = ALLOCATE_OBJ(vm, ObjList, OBJ_LIST);
     initValueArray(&list->values);
@@ -262,6 +266,20 @@ ObjString *takeString(DictuVM *vm, char *chars, int length) {
     return allocateString(vm, chars, length, hash);
 }
 
+ObjString *takeStringWithLen(DictuVM *vm, char *chars, int length, int character_len) {
+    uint32_t hash = hashString(chars, length);
+    ObjString *interned = tableFindString(&vm->strings, chars, length,
+                                          hash);
+    if (interned != NULL) {
+        FREE_ARRAY(vm, char, chars, length + 1);
+        return interned;
+    }
+
+    // Ensure terminating char is present
+    chars[length] = '\0';
+    return allocateStringWithLen(vm, chars, length, hash, character_len);
+}
+
 ObjString *copyString(DictuVM *vm, const char *chars, int length) {
     uint32_t hash = hashString(chars, length);
     ObjString *interned = tableFindString(&vm->strings, chars, length,
@@ -274,6 +292,18 @@ ObjString *copyString(DictuVM *vm, const char *chars, int length) {
     return allocateString(vm, heapChars, length, hash);
 }
 
+ObjString *copyStringWithLen(DictuVM *vm, const char *chars, int length, int character_len) {
+    uint32_t hash = hashString(chars, length);
+    ObjString *interned = tableFindString(&vm->strings, chars, length,
+                                          hash);
+    if (interned != NULL) return interned;
+
+    char *heapChars = ALLOCATE(vm, char, length + 1);
+    memcpy(heapChars, chars, length);
+    heapChars[length] = '\0';
+    return allocateStringWithLen(vm, heapChars, length, hash, character_len);
+}
+
 ObjUpvalue *newUpvalue(DictuVM *vm, Value *slot) {
     ObjUpvalue *upvalue = ALLOCATE_OBJ(vm, ObjUpvalue, OBJ_UPVALUE);
     upvalue->closed = NIL_VAL;
diff --git a/src/vm/object.h b/src/vm/object.h
index 1eeb01d7b..212876d7a 100644
--- a/src/vm/object.h
+++ b/src/vm/object.h
@@ -271,8 +271,10 @@ ObjInstance *newInstance(DictuVM *vm, ObjClass *klass);
 ObjNative *newNative(DictuVM *vm, NativeFn function);
 
 ObjString *takeString(DictuVM *vm, char *chars, int length);
+ObjString *takeStringWithLen(DictuVM *vm, char *chars, int length, int character_len);
 
 ObjString *copyString(DictuVM *vm, const char *chars, int length);
+ObjString *copyStringWithLen(DictuVM *vm, const char *chars, int length, int character_len);
 
 ObjList *newList(DictuVM *vm);
 
diff --git a/src/vm/vm.c b/src/vm/vm.c
index 1dc07fb95..3a63692cf 100644
--- a/src/vm/vm.c
+++ b/src/vm/vm.c
@@ -10,6 +10,7 @@
 #include "object.h"
 #include "memory.h"
 #include "vm.h"
+#include "utf8.h"
 #include "util.h"
 #include "error_lib/error.h"
 #include "datatypes/number.h"
@@ -1898,14 +1899,26 @@ static DictuInterpretResult run(DictuVM *vm) {
 
                 case OBJ_STRING: {
                     ObjString *string = AS_STRING(subscriptValue);
+                    int len = string->character_len == -1 ? string->length : string->character_len;
                     int index = AS_NUMBER(indexValue);
 
                     // Allow negative indexes
                     if (index < 0)
-                        index = string->length + index;
-
-                    if (index >= 0 && index < string->length) {
-                        ObjString *newString = copyString(vm, &string->chars[index], 1);
+                        index = len + index;
+
+                    if (index >= 0 && index < len) {
+                        ObjString* newString;
+                        if(string->character_len != -1) {
+                            utf8_int32_t ch;
+                            char* ptr = string->chars;
+                            for(size_t i = 0; i <= (size_t)index; i++)
+                                ptr = utf8codepoint(ptr, &ch);
+                            size_t cpSize = utf8codepointsize(ch);
+                            newString = copyString(vm, ptr - cpSize, cpSize);
+                            
+                        } else {
+                            newString = copyString(vm, &string->chars[index], 1);
+                        }
                         pop(vm);
                         pop(vm);
                         push(vm, OBJ_VAL(newString));
@@ -2103,16 +2116,17 @@ static DictuInterpretResult run(DictuVM *vm) {
 
                 case OBJ_STRING: {
                     ObjString *string = AS_STRING(objectValue);
+                    int len = string->character_len == -1 ? string->length : string->character_len;
 
                     if (IS_EMPTY(sliceEndIndex)) {
-                        indexEnd = string->length;
+                        indexEnd = len;
                     } else {
                         indexEnd = AS_NUMBER(sliceEndIndex);
 
-                        if (indexEnd > string->length) {
-                            indexEnd = string->length;
+                        if (indexEnd > len) {
+                            indexEnd = len;
                         }  else if (indexEnd < 0) {
-                            indexEnd = string->length + indexEnd;
+                            indexEnd = len + indexEnd;
                         }
                     }
 
@@ -2120,7 +2134,20 @@ static DictuInterpretResult run(DictuVM *vm) {
                     if (indexStart > indexEnd) {
                         returnVal = OBJ_VAL(copyString(vm, "", 0));
                     } else {
-                        returnVal = OBJ_VAL(copyString(vm, string->chars + indexStart, indexEnd - indexStart));
+                        if(string->character_len != -1) {
+                            utf8_int32_t ch;
+                            char* ptr = string->chars;
+                        // first we need to advance by the skip;
+                            for(size_t i = 0; i < (size_t)indexStart; i++)
+                                ptr = utf8codepoint(ptr, &ch);
+                            char* ptrEnd = ptr;
+                            for(size_t i = 0; i < (size_t)(indexEnd - indexStart); i++)
+                                ptrEnd = utf8codepoint(ptrEnd, &ch);
+                            returnVal = OBJ_VAL(copyString(vm, ptr, ptrEnd - ptr));
+                        } else {
+                            returnVal = OBJ_VAL(copyString(vm, string->chars + indexStart, indexEnd - indexStart));
+                        }
+
                     }
                     break;
                 }

From cb1198dd95a4784aed8d4ddd1cdce47b125d681c Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Sun, 27 Oct 2024 00:36:11 +0200
Subject: [PATCH 06/31] use the length version of the take/copyString functions

---
 src/vm/datatypes/strings.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 4f4bac699..d7542ed65 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -449,7 +449,7 @@ static Value lowerString(DictuVM *vm, int argCount, Value *args) {
     utf8lwr(temp);
     temp[string->length] = '\0';
 
-    return OBJ_VAL(takeString(vm, temp, string->length));
+    return OBJ_VAL(takeStringWithLen(vm, temp, string->length, string->character_len));
 }
 
 static Value upperString(DictuVM *vm, int argCount, Value *args) {
@@ -469,7 +469,7 @@ static Value upperString(DictuVM *vm, int argCount, Value *args) {
     temp[string->length] = 0x00;
     utf8upr(temp);
 
-    return OBJ_VAL(takeString(vm, temp, string->length));
+    return OBJ_VAL(takeStringWithLen(vm, temp, string->length, string->character_len));
 }
 
 static Value startsWithString(DictuVM *vm, int argCount, Value *args) {
@@ -686,7 +686,7 @@ static Value titleString(DictuVM *vm, int argCount, Value *args) {
 
     temp[string->length] = '\0';
 
-    return OBJ_VAL(takeString(vm, temp, string->length));
+    return OBJ_VAL(takeStringWithLen(vm, temp, string->length, string->character_len));
 }
 
 static Value repeatString(DictuVM *vm, int argCount, Value *args) {

From e6ee09fdde7b87defc203dea627c1b8cdd01d013 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Sun, 27 Oct 2024 00:39:14 +0200
Subject: [PATCH 07/31] need to cast this

---
 src/vm/object.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vm/object.c b/src/vm/object.c
index 04cf28524..6324d02cd 100644
--- a/src/vm/object.c
+++ b/src/vm/object.c
@@ -163,7 +163,7 @@ static ObjString *allocateStringWithLen(DictuVM *vm, char *chars, int length,
 
 static ObjString *allocateString(DictuVM *vm, char *chars, int length,
                                  uint32_t hash) {
-    int character_len = utf8valid(chars) == 0 ? utf8len(chars) : -1;
+    int character_len = utf8valid(chars) == 0 ? (int)utf8len(chars) : -1;
     return allocateStringWithLen(vm, chars, length, hash, character_len);
 }
 

From 8269774d209de0e08d3b689dac59565d17c18d6c Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Sun, 27 Oct 2024 01:51:02 +0200
Subject: [PATCH 08/31] oh my god it took me forever to figure this out

---
 src/cli/CMakeLists.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt
index 588a3ff33..5ed8e89b0 100644
--- a/src/cli/CMakeLists.txt
+++ b/src/cli/CMakeLists.txt
@@ -9,3 +9,7 @@ add_compile_definitions(USE_UTF8)
 add_executable(dictu ${DICTU_CLI_SRC})
 target_include_directories(dictu PUBLIC ${INCLUDE_DIR})
 target_link_libraries(dictu dictu_api_static)
+if(LINUX AND CMAKE_C_COMPILER_ID STREQUAL "Clang")
+    # Clang needs this otherwise ld will fail on linux since the lib is build with -flto
+    set(CMAKE_C_FLAGS_RELEASE "-flto")
+endif()
\ No newline at end of file

From ce27a065ae25498f29ebebaff2969ce92406805a Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Sun, 27 Oct 2024 03:10:40 +0100
Subject: [PATCH 09/31] add isValidUtf8

---
 src/vm/datatypes/strings.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index d7542ed65..57ba55832 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -852,12 +852,20 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
     return OBJ_VAL(takeString(vm, temp, utf8size_lazy(temp)));
 }
 
+static Value isValidUtf8(DictuVM* vm, int argCount, Value* args) {
+    UNUSED(vm);
+    UNUSED(argCount);
+    ObjString* string = AS_STRING(args[0]);
+    return BOOL_VAL(string->character_len != -1);
+}
+
 void declareStringMethods(DictuVM *vm) {
     // Note(Liz3): We need functions from the c stdlib for iswalpha, iswlower, iswupper(the utf8.c 
     // library functions do not work)
     setlocale(LC_ALL, "en_US.UTF-8");
     defineNative(vm, &vm->stringMethods, "len", lenString);
     defineNative(vm, &vm->stringMethods, "byteLen", byteLenString);
+    defineNative(vm, &vm->stringMethods, "isValidUtf8", isValidUtf8);
     defineNative(vm, &vm->stringMethods, "toNumber", toNumberString);
     defineNative(vm, &vm->stringMethods, "format", formatString);
     defineNative(vm, &vm->stringMethods, "split", splitString);

From 571c73e1dfea7d9fd9c224c96871751df838442d Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Sun, 27 Oct 2024 22:03:30 +0100
Subject: [PATCH 10/31] tests: add unicode slicing and indexing tests

---
 tests/strings/slicing.du   | 27 +++++++++++++++++++++++++++
 tests/strings/subscript.du | 13 +++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/tests/strings/slicing.du b/tests/strings/slicing.du
index 717119a8f..2d8d2611c 100644
--- a/tests/strings/slicing.du
+++ b/tests/strings/slicing.du
@@ -34,6 +34,33 @@ class TestStringSlicing < UnitTest {
         this.assertEquals(x[2:4], "ct");
         this.assertEquals(x[2:3], "c");
     }
+    testStringSliceUnicode() {
+        const x = "😀🌍🐦aä";
+        this.assertEquals(x[0:x.len()], x);
+        this.assertEquals(x[0:], x);
+        this.assertEquals(x[1:], "🌍🐦aä");
+        this.assertEquals(x[2:], "🐦aä");
+        this.assertEquals(x[3:], "aä");
+        this.assertEquals(x[4:], "ä");
+        this.assertEquals(x[5:], "");
+
+        this.assertEquals(x[:5], "😀🌍🐦aä");
+        this.assertEquals(x[:4], "😀🌍🐦a");
+        this.assertEquals(x[:3], "😀🌍🐦");
+        this.assertEquals(x[:2], "😀🌍");
+        this.assertEquals(x[:1], "😀");
+        this.assertEquals(x[:0], "");
+
+        this.assertEquals(x[1:4], "🌍🐦a");
+        this.assertEquals(x[1:3], "🌍🐦");
+        this.assertEquals(x[1:2], "🌍");
+        this.assertEquals(x[1:1], "");
+
+        this.assertEquals(x[2:5], "🐦aä");
+        this.assertEquals(x[2:4], "🐦a");
+        this.assertEquals(x[2:3], "🐦");
+        this.assertEquals(x[:x.len()], x);
+    }
 }
 
 TestStringSlicing().run();
\ No newline at end of file
diff --git a/tests/strings/subscript.du b/tests/strings/subscript.du
index eae3a39dc..f189a3345 100644
--- a/tests/strings/subscript.du
+++ b/tests/strings/subscript.du
@@ -18,6 +18,19 @@ class TestStringSubscript < UnitTest {
         this.assertEquals(string[-2], "t");
         this.assertEquals(string[-3], "a");
 
+        for (var i = 0; i < string.len(); i += 1) {
+            this.assertEquals(string[i], stringList[i]);
+        }
+    }
+    testStringSubscriptUnicode() {
+        const string = "😀🌍🐦aä";
+        const stringList = ["😀", "🌍", "🐦", "a", "ä"];
+
+        this.assertEquals(string[0], "😀");
+        this.assertEquals(string[1], "🌍");
+        this.assertEquals(string[2], "🐦");
+        this.assertEquals(string[-1], "ä");
+
         for (var i = 0; i < string.len(); i += 1) {
             this.assertEquals(string[i], stringList[i]);
         }

From a98d2d081959a820f72b0e5a2662b858ca195d69 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Mon, 28 Oct 2024 17:33:44 +0100
Subject: [PATCH 11/31] fix: only run index and slice utf8 version if there are
 multibyte characters

---
 src/vm/vm.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/vm/vm.c b/src/vm/vm.c
index 3a63692cf..3caef3d8d 100644
--- a/src/vm/vm.c
+++ b/src/vm/vm.c
@@ -1908,14 +1908,14 @@ static DictuInterpretResult run(DictuVM *vm) {
 
                     if (index >= 0 && index < len) {
                         ObjString* newString;
-                        if(string->character_len != -1) {
+                        if(string->character_len != -1 && string->character_len != string->length) {
                             utf8_int32_t ch;
                             char* ptr = string->chars;
                             for(size_t i = 0; i <= (size_t)index; i++)
                                 ptr = utf8codepoint(ptr, &ch);
                             size_t cpSize = utf8codepointsize(ch);
                             newString = copyString(vm, ptr - cpSize, cpSize);
-                            
+
                         } else {
                             newString = copyString(vm, &string->chars[index], 1);
                         }
@@ -2134,7 +2134,7 @@ static DictuInterpretResult run(DictuVM *vm) {
                     if (indexStart > indexEnd) {
                         returnVal = OBJ_VAL(copyString(vm, "", 0));
                     } else {
-                        if(string->character_len != -1) {
+                        if(string->character_len != -1 && string->character_len != string->length) {
                             utf8_int32_t ch;
                             char* ptr = string->chars;
                         // first we need to advance by the skip;

From e8a3d35615626ec5c05f3048ce687c219fb33eef Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Tue, 29 Oct 2024 21:40:42 +0100
Subject: [PATCH 12/31] fix: correctly reimplement findLast

---
 src/vm/datatypes/strings.c | 53 ++++++++++++++------------------------
 1 file changed, 19 insertions(+), 34 deletions(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 57ba55832..8189c092f 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -328,42 +328,27 @@ static Value findLastString(DictuVM *vm, int argCount, Value *args) {
            return EMPTY_VAL;
        }
     }
-
-    const char *str = AS_CSTRING(args[0]);
-    const char *ss = AS_CSTRING(args[1]);
-    const char *p = str;
-    int found = !*ss;
-
-    if (!found) {
-        while (*p) {
-            ++p;
-        }
-
-        const char *q = ss;
-        while (*q) {
-            ++q;
-        }
-
-        while (!found && !(p-str < q-ss)) {
-            const char *s = p;
-            const char *t = q;
-
-            while (t != ss && *(s-1) == *(t-1)) {
-                --s;
-                --t;
-            }
-
-            found = t == ss;
-
-            if (found) {
-                p = s;
-            } else {
-                --p;
-            }
-        }
+    utf8_int32_t cp;
+    ObjString *str = AS_STRING(args[0]);
+    ObjString *ss = AS_STRING(args[1]);
+    if(ss->length > str->length || ss->character_len > str->character_len)
+        return NUMBER_VAL(-1);
+
+    char* ptr = str->chars;
+    int index = str->character_len - ss->character_len;
+    for(int i = 0; i< index; i++)
+          ptr = utf8codepoint(ptr, &cp);
+
+    while(true) {
+        if(utf8str(ptr, ss->chars) == ptr)
+            return NUMBER_VAL(index);
+        index--;
+        if(index < 0)
+            break;
+        ptr = utf8rcodepoint(ptr, &cp);
     }
 
-    return NUMBER_VAL(found ? p-str : -1);
+    return NUMBER_VAL( -1);
 }
 
 static Value replaceString(DictuVM *vm, int argCount, Value *args) {

From 742c89fb9f98c7a36f62f37c23b89fbb13733ed0 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Tue, 29 Oct 2024 21:41:12 +0100
Subject: [PATCH 13/31] tests: implement unicode tests for other string
 functions

---
 tests/strings/collapseSpaces.du | 10 ++++++++++
 tests/strings/concat.du         | 10 ++++++++++
 tests/strings/contains.du       |  6 ++++++
 tests/strings/count.du          |  6 ++++++
 tests/strings/endsWith.du       |  7 +++++++
 tests/strings/findLast.du       |  5 +++++
 tests/strings/format.du         |  4 ++++
 tests/strings/lower.du          |  4 ++++
 tests/strings/repeat.du         |  5 +++++
 tests/strings/replace.du        |  5 +++++
 tests/strings/startsWith.du     |  1 +
 tests/strings/strip.du          | 13 ++++++++++++-
 tests/strings/title.du          |  7 +++++++
 tests/strings/toBool.du         |  3 +++
 tests/strings/upper.du          |  4 ++++
 tests/strings/wordCount.du      |  4 ++++
 tests/strings/wrap.du           |  5 +++++
 17 files changed, 98 insertions(+), 1 deletion(-)

diff --git a/tests/strings/collapseSpaces.du b/tests/strings/collapseSpaces.du
index d885ee3ff..1dcc47ddd 100644
--- a/tests/strings/collapseSpaces.du
+++ b/tests/strings/collapseSpaces.du
@@ -15,6 +15,16 @@ class TestStringCollapseSpaces < UnitTest {
         this.assertEquals(res, expected);
         this.assertNotEquals(testString, expected);
     }
+
+    testStringCollapseSpacesUnicode() {
+        const x = "😀🌍🐦aä";
+        const testString = "This  is   🌍   a huge string🐦  🐦 of a       lot of spaces.";
+        const expected  = "This is 🌍 a huge string🐦 🐦 of a lot of spaces.";
+        const res = testString.collapseSpaces();
+        this.assertEquals(res, expected);
+        this.assertNotEquals(testString, expected);
+    }
+
 }
 
 TestStringCollapseSpaces().run();
diff --git a/tests/strings/concat.du b/tests/strings/concat.du
index 9518698ca..fed763bf8 100644
--- a/tests/strings/concat.du
+++ b/tests/strings/concat.du
@@ -15,6 +15,16 @@ class TestStringConcat < UnitTest {
 
         this.assertEquals(x, "Dictu is great!");
     }
+    testStringConcatUnicode() {
+        this.assertEquals("😅" + " 😅 " + "😅", "😅 😅 😅");
+
+        const x = "Dictu" +
+            " 😊 " +
+            "great❗";
+
+        this.assertEquals(x, "Dictu 😊 great❗");
+    }
+
 }
 
 TestStringConcat().run();
diff --git a/tests/strings/contains.du b/tests/strings/contains.du
index 557cc0b92..48b9eabd8 100644
--- a/tests/strings/contains.du
+++ b/tests/strings/contains.du
@@ -17,6 +17,12 @@ class TestStringContains < UnitTest {
         this.assertTruthy("1Dictu is great!1".contains("1"));
         this.assertTruthy(("1Dictu " + "is great!").contains("1"));
     }
+    testStringContainsUnicode() {
+        this.assertTruthy(("❗Dictu " + "is great!").contains("❗"));
+        this.assertFalsey("Dictu is great!".contains("❗"));
+        this.assertTruthy("Dictu is ♨️ right?️".contains("♨️")); // ♨️ is two characters!
+
+    }
 }
 
 TestStringContains().run();
\ No newline at end of file
diff --git a/tests/strings/count.du b/tests/strings/count.du
index 873e57147..210ca3f80 100644
--- a/tests/strings/count.du
+++ b/tests/strings/count.du
@@ -16,6 +16,12 @@ class TestStringCount < UnitTest {
         this.assertEquals("Dictu is great! Dictu is great!".count("1234"), 0);
         this.assertEquals("Dictu is great! Dictu is great!".count("!"), 2);
     }
+    testStringCountUnicode() {
+        this.assertEquals("Dictu is 😎! Dictu is 😎!".count("Dictu is 😎!"), 2);
+        this.assertEquals("Dictu is great❗".count("❗"), 1);
+        this.assertEquals("Dictu is great❕ Dictu is great❕".count("❗"), 0);
+
+    }
 }
 
 TestStringCount().run();
\ No newline at end of file
diff --git a/tests/strings/endsWith.du b/tests/strings/endsWith.du
index 5447b9d64..6f8085ac8 100644
--- a/tests/strings/endsWith.du
+++ b/tests/strings/endsWith.du
@@ -19,6 +19,13 @@ class TestStringEndsWith < UnitTest {
         this.assertFalsey("test".endsWith("!@£$%"));
         this.assertFalsey("test".endsWith("123456789"));
     }
+    testStringEndsWithUnicode() {
+        this.assertTruthy("🌐😁⚓🌍".endsWith("🌍"));
+        this.assertTruthy("🌐😁⚓🌍".endsWith("⚓🌍"));
+        this.assertTruthy("🌐😁⚓🌍".endsWith("🌐😁⚓🌍"));
+        this.assertFalsey("🌐😁⚓🌍".endsWith("😅"));
+        this.assertFalsey("🌐😁⚓🌍".endsWith("🌐😁⚓🌍😅"));
+    }
 }
 
 TestStringEndsWith().run();
\ No newline at end of file
diff --git a/tests/strings/findLast.du b/tests/strings/findLast.du
index 1f7d04ea1..8eed13451 100644
--- a/tests/strings/findLast.du
+++ b/tests/strings/findLast.du
@@ -12,6 +12,11 @@ class TestStringLastIndexOf < UnitTest {
         this.assertEquals("woolly woolly mammoth".findLast("woolly"), 7);
         this.assertEquals("mammoth".findLast("woolly"), -1);
     }
+    testStringLowerUnicode() {
+        this.assertEquals("🌐😁⚓🌍 🌐😁⚓🌍 mammoth".findLast("🌐😁⚓🌍"), 5);
+        this.assertEquals("🌐😁⚓🌍".findLast("😅😅"), -1);
+
+    }
 }
 
 TestStringLastIndexOf().run();
\ No newline at end of file
diff --git a/tests/strings/format.du b/tests/strings/format.du
index 4d186c542..7375e1d43 100644
--- a/tests/strings/format.du
+++ b/tests/strings/format.du
@@ -21,6 +21,10 @@ class TestStringFormat < UnitTest {
         this.assertEquals("{}".format({"test": 10, "aaa": 10}), '{"aaa": 10, "test": 10}');
         this.assertEquals("{} {} {}".format(test, Test, Trait), '<fn test> <Cls Test> <Trait Trait>');
     }
+    testStringFormatUnicode() {
+        this.assertEquals("World: {} is {}".format("🌍", "😎"), "World: 🌍 is 😎");
+        this.assertEquals("hello {}".format(["🌐😁⚓🌍"]), "hello [\"🌐😁⚓🌍\"]");
+    }
 }
 
 TestStringFormat().run();
\ No newline at end of file
diff --git a/tests/strings/lower.du b/tests/strings/lower.du
index 9f7f58857..3fd091d5e 100644
--- a/tests/strings/lower.du
+++ b/tests/strings/lower.du
@@ -16,6 +16,10 @@ class TestStringLower < UnitTest {
         this.assertEquals("12Dictu45".lower(), "12dictu45");
         this.assertEquals("!@£$%^&*".lower(), "!@£$%^&*");
     }
+    testStringLowerUnicode() {
+        this.assertEquals("ÄÜÖ".lower(), "äüö");
+        this.assertEquals("DICTU🌍".lower(), "dictu🌍");
+    }
 }
 
 TestStringLower().run();
\ No newline at end of file
diff --git a/tests/strings/repeat.du b/tests/strings/repeat.du
index 60b6f8462..59bbdfcd9 100644
--- a/tests/strings/repeat.du
+++ b/tests/strings/repeat.du
@@ -12,6 +12,11 @@ class TestStringRepeat < UnitTest {
         this.assertEquals("ba" + "na".repeat(2), "banana");
         this.assertEquals("ha".repeat(6), "hahahahahaha");
     }
+    testStringRepeatUnicode() {
+        this.assertEquals("⌛" + "⏳".repeat(2), "⌛⏳⏳");
+        this.assertEquals("📃".repeat(5), "📃📃📃📃📃");
+    }
+
 }
 
 TestStringRepeat().run();
diff --git a/tests/strings/replace.du b/tests/strings/replace.du
index e8a215e5e..d00598e61 100644
--- a/tests/strings/replace.du
+++ b/tests/strings/replace.du
@@ -21,6 +21,11 @@ class TestStringReplace < UnitTest {
         this.assertEquals("test".replace("t", "123456789123456789123456789123456789"),
             "123456789123456789123456789123456789es123456789123456789123456789123456789");
     }
+    testStringReplaceUnicode() {
+        this.assertEquals("😀".replace("😀", "😅"), "😅");
+        this.assertEquals("😀😀".replace("😀", "😀😀"), "😀😀😀😀");
+        this.assertEquals("❕string❗".replace("❗", "❕❕"), "❕string❕❕");
+    }
 }
 
 TestStringReplace().run();
\ No newline at end of file
diff --git a/tests/strings/startsWith.du b/tests/strings/startsWith.du
index c145034a6..d74d39809 100644
--- a/tests/strings/startsWith.du
+++ b/tests/strings/startsWith.du
@@ -21,6 +21,7 @@ class TestStringStartsWith < UnitTest {
     }
     testStringStartsWithUnicode() {
         this.assertTruthy("💻test".startsWith("💻t"));
+        this.assertTruthy("💻test".startsWith("💻test"));
         this.assertFalsey("💻test".startsWith("💻te💻"));
     }
 }
diff --git a/tests/strings/strip.du b/tests/strings/strip.du
index 3297a1a47..3880d61ec 100644
--- a/tests/strings/strip.du
+++ b/tests/strings/strip.du
@@ -27,7 +27,7 @@ class TestStringStrip < UnitTest {
         this.assertEquals("  Dictu is great!".rightStrip().len(), 17);
         this.assertEquals("  Dictu is great!  ".rightStrip().len(), 17);
     }
-    
+
     testStringStrip() {
         this.assertEquals("Dictu is great!  ".strip(), "Dictu is great!");
         this.assertEquals("  Dictu is great!".strip(), "Dictu is great!");
@@ -36,6 +36,17 @@ class TestStringStrip < UnitTest {
         this.assertEquals("  Dictu is great!".strip().len(), 15);
         this.assertEquals("  Dictu is great!  ".strip().len(), 15);
     }
+
+    testStringStripUnicode() {
+        this.assertEquals("🐦Dictu is great🐦  ".strip(), "🐦Dictu is great🐦");
+        this.assertEquals("  🐦Dictu is great!🐦".strip(), "🐦Dictu is great!🐦");
+        this.assertEquals("  🐦Dictu is great!🐦  ".strip(), "🐦Dictu is great!🐦");
+        this.assertEquals("Dictu is great🐦  ".strip().len(), 15);
+        this.assertEquals("  Dictu is great🐦".strip().len(), 15);
+        this.assertEquals("  🐦Dictu is great🐦  ".strip().len(), 16);
+    }
+
+
 }
 
 TestStringStrip().run();
\ No newline at end of file
diff --git a/tests/strings/title.du b/tests/strings/title.du
index ecd1c85e1..b17bd47a9 100644
--- a/tests/strings/title.du
+++ b/tests/strings/title.du
@@ -17,6 +17,13 @@ class TestStringTitle < UnitTest {
         this.assertEquals("!@£$%^&*".title(), "!@£$%^&*");
         this.assertEquals("once upon a time".title(), "Once Upon A Time");
     }
+    testStringTitleUnicode() {
+        this.assertEquals("über".title(), "Über");
+        this.assertEquals("üBer".title(), "Über");
+        this.assertEquals("👻👻".title(), "👻👻");
+        this.assertEquals("👻👻 test".title(), "👻👻 Test");
+        this.assertEquals("👻👻 test üBer".title(), "👻👻 Test Über");
+    }
 }
 
 TestStringTitle().run();
\ No newline at end of file
diff --git a/tests/strings/toBool.du b/tests/strings/toBool.du
index 719b035f6..5c6f8928d 100644
--- a/tests/strings/toBool.du
+++ b/tests/strings/toBool.du
@@ -12,6 +12,9 @@ class TestStringToBool < UnitTest {
         this.assertTruthy("10".toBool());
         this.assertFalsey("".toBool());
     }
+    testStringToBoolUnicode() {
+        this.assertTruthy("💬".toBool());
+    }
 }
 
 TestStringToBool().run();
\ No newline at end of file
diff --git a/tests/strings/upper.du b/tests/strings/upper.du
index a7012e976..40a444092 100644
--- a/tests/strings/upper.du
+++ b/tests/strings/upper.du
@@ -16,6 +16,10 @@ class TestStringUpper < UnitTest {
         this.assertEquals("12Dictu45".upper(), "12DICTU45");
         this.assertEquals("!@£$%^&*".upper(), "!@£$%^&*");
     }
+    testStringUpperUnicode() {
+        this.assertEquals("üäö".upper(), "ÜÄÖ");
+        this.assertEquals("ü🌐test".upper(), "Ü🌐TEST");
+    }
 }
 
 TestStringUpper().run();
\ No newline at end of file
diff --git a/tests/strings/wordCount.du b/tests/strings/wordCount.du
index 6f93cfde1..614321f8b 100644
--- a/tests/strings/wordCount.du
+++ b/tests/strings/wordCount.du
@@ -14,6 +14,10 @@ class TestStringWordCount < UnitTest {
         this.assertEquals("This is a sentence".wordCount(), 4);
         this.assertEquals("This is an even longer sentence".wordCount(), 6);
     }
+    testStringWordCountUnicode() {
+      this.assertEquals("Dictu is cool😎".wordCount(), 3);
+      this.assertEquals("WOrld🌐 üäö".wordCount(), 2);
+    }
 }
 
 TestStringWordCount().run();
diff --git a/tests/strings/wrap.du b/tests/strings/wrap.du
index 442b04652..0de75e361 100644
--- a/tests/strings/wrap.du
+++ b/tests/strings/wrap.du
@@ -16,6 +16,11 @@ class TestStringWrap < UnitTest {
         const idx = res.find("\n");
         this.assertTruthy(res.find("\n") <= this.maxLen);
     }
+    testStringWrapUnicode() {
+        const _len = 5;
+        const wrapped = "🌐test 🐦🐦🐦 test344🐦".wrap(_len);
+        this.assertEquals(wrapped, "🌐test\n🐦🐦🐦\ntest344🐦");
+    }
 }
 
 TestStringWrap().run();

From c253114007fb714bd9a41bdedd8d04c642a98615 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Tue, 29 Oct 2024 22:01:17 +0100
Subject: [PATCH 14/31] fix errors

---
 src/vm/datatypes/strings.c      | 7 ++++---
 tests/strings/collapseSpaces.du | 1 -
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 8189c092f..3451137a6 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -783,7 +783,7 @@ static Value collapseSpacesString(DictuVM *vm, int argCount, Value *args) {
         utf8_int32_t cp;
         char* n = utf8codepoint(ptr, &cp);
         
-        if (!isspace(cp) || (i > 0 && !isspace(last))) {
+        if (!iswspace(cp) || (i > 0 && !iswspace(last))) {
             utf8catcodepoint(dest+targetLen, cp, string->length - targetLen);
             targetLen += utf8codepointsize(cp);
         }
@@ -812,10 +812,11 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
        return EMPTY_VAL;
    }
     char *temp = ALLOCATE(vm, char, string->length + 1);
+    temp[string->length] = 0x00;
     memcpy(temp, string->chars, string->length);
     int len = AS_NUMBER(args[1]);
 
-    int last = 0;
+    int last = -1;
     int count = 0;
     char*ptr = temp;
     for (int cur = 0; cur < string->character_len; cur++, count++) {
@@ -827,7 +828,7 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
             last = ptr - temp;
         } 
 
-        if (count >= len) {
+        if (count >= len && last != -1) {
             temp[last] = '\n';
             count = 0;
         }
diff --git a/tests/strings/collapseSpaces.du b/tests/strings/collapseSpaces.du
index 1dcc47ddd..766b5f42d 100644
--- a/tests/strings/collapseSpaces.du
+++ b/tests/strings/collapseSpaces.du
@@ -17,7 +17,6 @@ class TestStringCollapseSpaces < UnitTest {
     }
 
     testStringCollapseSpacesUnicode() {
-        const x = "😀🌍🐦aä";
         const testString = "This  is   🌍   a huge string🐦  🐦 of a       lot of spaces.";
         const expected  = "This is 🌍 a huge string🐦 🐦 of a lot of spaces.";
         const res = testString.collapseSpaces();

From 8206e4e840ff9be8f744a185748b912c57182c7e Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Wed, 30 Oct 2024 22:06:54 +0100
Subject: [PATCH 15/31] fix: when executing a child process, do not waitpid
 before reading

Trying to waitpid before reading the piped content will lead to a blocked pipe and
get the child process "stuck". So it should be read BEFORE waitpid is called.
Further there was a bug within the reallocation logic leading to heap corruption
because the comparison size did not include the just read chunk, leading to heap corruption,
which itself led to reallocate failiing.
---
 src/optionals/process.c | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/optionals/process.c b/src/optionals/process.c
index ec7de1f52..75be624f2 100644
--- a/src/optionals/process.c
+++ b/src/optionals/process.c
@@ -201,13 +201,6 @@ static Value executeReturnOutput(DictuVM* vm, ObjList* argList) {
 
     close(fd[1]);
 
-    int status = 0;
-    waitpid(pid, &status, 0);
-
-    if (WIFEXITED(status) && (status = WEXITSTATUS(status)) != 0) {
-        ERROR_RESULT;
-    }
-
     int size = 1024;
     char* output = ALLOCATE(vm, char, size);
     char buffer[1024];
@@ -215,7 +208,7 @@ static Value executeReturnOutput(DictuVM* vm, ObjList* argList) {
     int numRead;
 
     while ((numRead = read(fd[0], buffer, 1024)) != 0) {
-        if (total >= size) {
+        if (total + numRead >= size) {
             output = GROW_ARRAY(vm, output, char, size, size * 3);
             size *= 3;
         }
@@ -224,6 +217,15 @@ static Value executeReturnOutput(DictuVM* vm, ObjList* argList) {
         total += numRead;
     }
 
+    close(fd[0]);
+
+    int status = 0;
+    waitpid(pid, &status, 0);
+
+    if (WIFEXITED(status) && (status = WEXITSTATUS(status)) != 0) {
+        ERROR_RESULT;
+    }
+
     output = SHRINK_ARRAY(vm, output, char, size, total + 1);
     output[total] = '\0';
 

From 218579faad287a32661dc09abb82103de4efec67 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Thu, 31 Oct 2024 01:04:32 +0100
Subject: [PATCH 16/31] length of this is 36

---
 src/optionals/uuid.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/optionals/uuid.c b/src/optionals/uuid.c
index f0e3da6e7..6f5be59f1 100644
--- a/src/optionals/uuid.c
+++ b/src/optionals/uuid.c
@@ -3,7 +3,7 @@
 #ifndef _WIN32
 static uuid_t uuid;
 
-#define UUID_STRING_LEN 37
+#define UUID_STRING_LEN 36
 
 static Value uuidGenerateNative(DictuVM *vm, int argCount, Value *args) {
     UNUSED(args);

From f85654b89eb35de3727bfa5cde97f8e72a7555bd Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Thu, 31 Oct 2024 05:45:57 +0100
Subject: [PATCH 17/31] reenable this

---
 tests/datetime/import.du | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/datetime/import.du b/tests/datetime/import.du
index bfcbafa9e..042d113be 100644
--- a/tests/datetime/import.du
+++ b/tests/datetime/import.du
@@ -6,4 +6,4 @@
 
 import "constants.du";
 import "strftime.du";
-//import "strptime.du";
+import "strptime.du";

From a676173d7f405341c2634d0b7d02cac604b66c7d Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Thu, 31 Oct 2024 05:48:49 +0100
Subject: [PATCH 18/31] UUIDS are length 36

---
 tests/uuid/generate.du       | 2 +-
 tests/uuid/generateRandom.du | 2 +-
 tests/uuid/generateTime.du   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/uuid/generate.du b/tests/uuid/generate.du
index 6dcc64458..65c0e41c4 100644
--- a/tests/uuid/generate.du
+++ b/tests/uuid/generate.du
@@ -16,7 +16,7 @@ class TestUUIDGenerate < UnitTest {
         this.assertTruthy(ret.success());
         const uuid = ret.unwrap();
         this.assertType(type(uuid), "string");
-        this.assertTruthy(uuid.byteLen() == 37);
+        this.assertTruthy(uuid.byteLen() == 36);
         print(uuid);
     }
 }
diff --git a/tests/uuid/generateRandom.du b/tests/uuid/generateRandom.du
index d3e6afcaf..f1844627d 100644
--- a/tests/uuid/generateRandom.du
+++ b/tests/uuid/generateRandom.du
@@ -16,7 +16,7 @@ class TestUUIDGenerateRandom < UnitTest {
         this.assertTruthy(ret.success());
         const uuid = ret.unwrap();
         this.assertType(type(uuid), "string");
-        this.assertTruthy(uuid.byteLen() == 37);
+        this.assertTruthy(uuid.byteLen() == 36);
     }
 }
 
diff --git a/tests/uuid/generateTime.du b/tests/uuid/generateTime.du
index 5cfa84b10..2f88e7810 100644
--- a/tests/uuid/generateTime.du
+++ b/tests/uuid/generateTime.du
@@ -16,7 +16,7 @@ class TestUUIDGenerateTime < UnitTest {
         this.assertTruthy(ret.success());
         const uuid = ret.unwrap();
         this.assertType(type(uuid), "string");
-        this.assertTruthy(uuid.byteLen() == 37);
+        this.assertTruthy(uuid.byteLen() == 36);
     }
 }
 

From 4f856d5dca96e62da30f7134cd0f74823c2f3aff Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Tue, 5 Nov 2024 23:03:24 +0100
Subject: [PATCH 19/31] fix: refactor scanner to use utf8, windows is confusing
 me

---
 src/vm/scanner.c | 73 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 47 insertions(+), 26 deletions(-)

diff --git a/src/vm/scanner.c b/src/vm/scanner.c
index 61f40480a..4dc702c66 100644
--- a/src/vm/scanner.c
+++ b/src/vm/scanner.c
@@ -1,6 +1,8 @@
 #include <string.h>
+#include <uchar.h>
 
 #include "common.h"
+#include "utf8.h"
 #include "scanner.h"
 
 void initScanner(Scanner *scanner, const char *source) {
@@ -10,17 +12,17 @@ void initScanner(Scanner *scanner, const char *source) {
     scanner->rawString = false;
 }
 
-static bool isAlpha(char c) {
+static bool isAlpha(char32_t c) {
     return (c >= 'a' && c <= 'z') ||
            (c >= 'A' && c <= 'Z') ||
            c == '_';
 }
 
-static bool isDigit(char c) {
+static bool isDigit(char32_t c) {
     return c >= '0' && c <= '9';
 }
 
-static bool isHexDigit(char c) {
+static bool isHexDigit(char32_t c) {
     return ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f') || (c == '_'));
 }
 
@@ -29,23 +31,31 @@ static bool isAtEnd(Scanner *scanner) {
 }
 
 static char advance(Scanner *scanner) {
-    scanner->current++;
-    return scanner->current[-1];
+    utf8_int32_t current;
+    scanner->current = utf8codepoint(scanner->current, &current);
+    return current;
 }
 
 static char peek(Scanner *scanner) {
-    return *scanner->current;
+    utf8_int32_t current;
+    utf8codepoint(scanner->current, &current);
+    return current;
 }
 
 static char peekNext(Scanner *scanner) {
     if (isAtEnd(scanner)) return '\0';
-    return scanner->current[1];
+    utf8_int32_t current;
+    const char* ptr = utf8codepoint(scanner->current, &current);
+    utf8codepoint(ptr, &current);
+    return current;
 }
 
-static bool match(Scanner *scanner, char expected) {
+static bool match(Scanner *scanner, char32_t expected) {
     if (isAtEnd(scanner)) return false;
-    if (*scanner->current != expected) return false;
-    scanner->current++;
+    utf8_int32_t current;
+    const char* ptr = utf8codepoint(scanner->current, &current);
+    if ((char32_t)current != expected) return false;
+    scanner->current = ptr;
     return true;
 }
 
@@ -70,7 +80,7 @@ static LangToken errorToken(Scanner *scanner, const char *message) {
 
 static void skipWhitespace(Scanner *scanner) {
     for (;;) {
-        char c = peek(scanner);
+        char32_t c = peek(scanner);
         switch (c) {
             case ' ':
             case '\r':
@@ -134,11 +144,22 @@ static LangTokenType checkKeyword(Scanner *scanner, int start, int length,
     return TOKEN_IDENTIFIER;
 }
 
+char32_t getCharacter(const char* ptr, int offset) {
+        utf8_int32_t current;
+
+    for(int i = 0; i < offset+1;i++) {
+         ptr = utf8codepoint(ptr, &current);
+
+    }
+    return current;
+}
+
 static LangTokenType identifierType(Scanner *scanner) {
-    switch (scanner->start[0]) {
+
+    switch (getCharacter(scanner->start, 0)) {
         case 'a':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 'b': {
                         return checkKeyword(scanner, 2, 6, "stract", TOKEN_ABSTRACT);
                     }
@@ -157,7 +178,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             return checkKeyword(scanner, 1, 4, "reak", TOKEN_BREAK);
         case 'c':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 'a':
 			return checkKeyword(scanner, 2, 2, "se", TOKEN_CASE);
                     case 'l':
@@ -166,7 +187,7 @@ static LangTokenType identifierType(Scanner *scanner) {
                         // Skip second char
                         // Skip third char
                         if (scanner->current - scanner->start > 3) {
-                            switch (scanner->start[3]) {
+                            switch (getCharacter(scanner->start, 3)) {
                                 case 't':
                                     return checkKeyword(scanner, 4, 4, "inue", TOKEN_CONTINUE);
                                 case 's':
@@ -179,7 +200,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 'd':
             if (scanner->current - scanner->start > 3) {
-                switch (scanner->start[3]) {
+                switch (getCharacter(scanner->start, 3)) {
                     case 'a':
                         return checkKeyword(scanner, 4, 3, "ult", TOKEN_DEFAULT);
                 }
@@ -187,7 +208,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             return checkKeyword(scanner, 1, 2, "ef", TOKEN_DEF);
         case 'e':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 'l':
                         return checkKeyword(scanner, 2, 2, "se", TOKEN_ELSE);
                     case 'n':
@@ -197,7 +218,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 'f':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 'a':
                         return checkKeyword(scanner, 2, 3, "lse", TOKEN_FALSE);
                     case 'o':
@@ -209,7 +230,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 'i':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 'f':
                         return checkKeyword(scanner, 2, 0, "", TOKEN_IF);
                     case 'm':
@@ -219,7 +240,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 'n':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 'o':
                         return checkKeyword(scanner, 2, 1, "t", TOKEN_NOT);
                     case 'i':
@@ -233,7 +254,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             return checkKeyword(scanner, 1, 6, "rivate", TOKEN_PRIVATE);
         case 'r':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 'e':
                         return checkKeyword(scanner, 2, 4, "turn", TOKEN_RETURN);
                 }
@@ -246,7 +267,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 's':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 't':
                         return checkKeyword(scanner, 2, 4, "atic", TOKEN_STATIC);
                     case 'u':
@@ -258,12 +279,12 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 't':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 'h':
                         return checkKeyword(scanner, 2, 2, "is", TOKEN_THIS);
                     case 'r':
                         if (scanner->current - scanner->start > 2) {
-                            switch (scanner->start[2]) {
+                            switch (getCharacter(scanner->start, 2)) {
                                 case 'u':
                                     return checkKeyword(scanner, 3, 1, "e", TOKEN_TRUE);
                                 case 'a':
@@ -279,7 +300,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             return checkKeyword(scanner, 1, 2, "ar", TOKEN_VAR);
         case 'w':
             if (scanner->current - scanner->start > 1) {
-                switch (scanner->start[1]) {
+                switch (getCharacter(scanner->start, 1)) {
                     case 'h':
                         return checkKeyword(scanner, 2, 3, "ile", TOKEN_WHILE);
                     case 'i':
@@ -366,7 +387,7 @@ LangToken scanToken(Scanner *scanner) {
 
     if (isAtEnd(scanner)) return makeToken(scanner, TOKEN_EOF);
 
-    char c = advance(scanner);
+    char32_t c = advance(scanner);
 
     if (isAlpha(c)) return identifier(scanner);
     if (isDigit(c)) return hexNumber(scanner);

From 2aed1ece67486708721b2caec636206b3519de89 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Wed, 6 Nov 2024 02:21:29 +0100
Subject: [PATCH 20/31] Revert "fix: refactor scanner to use utf8, windows is
 confusing me"

This reverts commit 4f856d5dca96e62da30f7134cd0f74823c2f3aff.
---
 src/vm/scanner.c | 73 +++++++++++++++++-------------------------------
 1 file changed, 26 insertions(+), 47 deletions(-)

diff --git a/src/vm/scanner.c b/src/vm/scanner.c
index 4dc702c66..61f40480a 100644
--- a/src/vm/scanner.c
+++ b/src/vm/scanner.c
@@ -1,8 +1,6 @@
 #include <string.h>
-#include <uchar.h>
 
 #include "common.h"
-#include "utf8.h"
 #include "scanner.h"
 
 void initScanner(Scanner *scanner, const char *source) {
@@ -12,17 +10,17 @@ void initScanner(Scanner *scanner, const char *source) {
     scanner->rawString = false;
 }
 
-static bool isAlpha(char32_t c) {
+static bool isAlpha(char c) {
     return (c >= 'a' && c <= 'z') ||
            (c >= 'A' && c <= 'Z') ||
            c == '_';
 }
 
-static bool isDigit(char32_t c) {
+static bool isDigit(char c) {
     return c >= '0' && c <= '9';
 }
 
-static bool isHexDigit(char32_t c) {
+static bool isHexDigit(char c) {
     return ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f') || (c == '_'));
 }
 
@@ -31,31 +29,23 @@ static bool isAtEnd(Scanner *scanner) {
 }
 
 static char advance(Scanner *scanner) {
-    utf8_int32_t current;
-    scanner->current = utf8codepoint(scanner->current, &current);
-    return current;
+    scanner->current++;
+    return scanner->current[-1];
 }
 
 static char peek(Scanner *scanner) {
-    utf8_int32_t current;
-    utf8codepoint(scanner->current, &current);
-    return current;
+    return *scanner->current;
 }
 
 static char peekNext(Scanner *scanner) {
     if (isAtEnd(scanner)) return '\0';
-    utf8_int32_t current;
-    const char* ptr = utf8codepoint(scanner->current, &current);
-    utf8codepoint(ptr, &current);
-    return current;
+    return scanner->current[1];
 }
 
-static bool match(Scanner *scanner, char32_t expected) {
+static bool match(Scanner *scanner, char expected) {
     if (isAtEnd(scanner)) return false;
-    utf8_int32_t current;
-    const char* ptr = utf8codepoint(scanner->current, &current);
-    if ((char32_t)current != expected) return false;
-    scanner->current = ptr;
+    if (*scanner->current != expected) return false;
+    scanner->current++;
     return true;
 }
 
@@ -80,7 +70,7 @@ static LangToken errorToken(Scanner *scanner, const char *message) {
 
 static void skipWhitespace(Scanner *scanner) {
     for (;;) {
-        char32_t c = peek(scanner);
+        char c = peek(scanner);
         switch (c) {
             case ' ':
             case '\r':
@@ -144,22 +134,11 @@ static LangTokenType checkKeyword(Scanner *scanner, int start, int length,
     return TOKEN_IDENTIFIER;
 }
 
-char32_t getCharacter(const char* ptr, int offset) {
-        utf8_int32_t current;
-
-    for(int i = 0; i < offset+1;i++) {
-         ptr = utf8codepoint(ptr, &current);
-
-    }
-    return current;
-}
-
 static LangTokenType identifierType(Scanner *scanner) {
-
-    switch (getCharacter(scanner->start, 0)) {
+    switch (scanner->start[0]) {
         case 'a':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 'b': {
                         return checkKeyword(scanner, 2, 6, "stract", TOKEN_ABSTRACT);
                     }
@@ -178,7 +157,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             return checkKeyword(scanner, 1, 4, "reak", TOKEN_BREAK);
         case 'c':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 'a':
 			return checkKeyword(scanner, 2, 2, "se", TOKEN_CASE);
                     case 'l':
@@ -187,7 +166,7 @@ static LangTokenType identifierType(Scanner *scanner) {
                         // Skip second char
                         // Skip third char
                         if (scanner->current - scanner->start > 3) {
-                            switch (getCharacter(scanner->start, 3)) {
+                            switch (scanner->start[3]) {
                                 case 't':
                                     return checkKeyword(scanner, 4, 4, "inue", TOKEN_CONTINUE);
                                 case 's':
@@ -200,7 +179,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 'd':
             if (scanner->current - scanner->start > 3) {
-                switch (getCharacter(scanner->start, 3)) {
+                switch (scanner->start[3]) {
                     case 'a':
                         return checkKeyword(scanner, 4, 3, "ult", TOKEN_DEFAULT);
                 }
@@ -208,7 +187,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             return checkKeyword(scanner, 1, 2, "ef", TOKEN_DEF);
         case 'e':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 'l':
                         return checkKeyword(scanner, 2, 2, "se", TOKEN_ELSE);
                     case 'n':
@@ -218,7 +197,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 'f':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 'a':
                         return checkKeyword(scanner, 2, 3, "lse", TOKEN_FALSE);
                     case 'o':
@@ -230,7 +209,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 'i':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 'f':
                         return checkKeyword(scanner, 2, 0, "", TOKEN_IF);
                     case 'm':
@@ -240,7 +219,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 'n':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 'o':
                         return checkKeyword(scanner, 2, 1, "t", TOKEN_NOT);
                     case 'i':
@@ -254,7 +233,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             return checkKeyword(scanner, 1, 6, "rivate", TOKEN_PRIVATE);
         case 'r':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 'e':
                         return checkKeyword(scanner, 2, 4, "turn", TOKEN_RETURN);
                 }
@@ -267,7 +246,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 's':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 't':
                         return checkKeyword(scanner, 2, 4, "atic", TOKEN_STATIC);
                     case 'u':
@@ -279,12 +258,12 @@ static LangTokenType identifierType(Scanner *scanner) {
             break;
         case 't':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 'h':
                         return checkKeyword(scanner, 2, 2, "is", TOKEN_THIS);
                     case 'r':
                         if (scanner->current - scanner->start > 2) {
-                            switch (getCharacter(scanner->start, 2)) {
+                            switch (scanner->start[2]) {
                                 case 'u':
                                     return checkKeyword(scanner, 3, 1, "e", TOKEN_TRUE);
                                 case 'a':
@@ -300,7 +279,7 @@ static LangTokenType identifierType(Scanner *scanner) {
             return checkKeyword(scanner, 1, 2, "ar", TOKEN_VAR);
         case 'w':
             if (scanner->current - scanner->start > 1) {
-                switch (getCharacter(scanner->start, 1)) {
+                switch (scanner->start[1]) {
                     case 'h':
                         return checkKeyword(scanner, 2, 3, "ile", TOKEN_WHILE);
                     case 'i':
@@ -387,7 +366,7 @@ LangToken scanToken(Scanner *scanner) {
 
     if (isAtEnd(scanner)) return makeToken(scanner, TOKEN_EOF);
 
-    char32_t c = advance(scanner);
+    char c = advance(scanner);
 
     if (isAlpha(c)) return identifier(scanner);
     if (isDigit(c)) return hexNumber(scanner);

From 24cd95875f596cab969dfbd960ae48cc3f42f857 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Wed, 6 Nov 2024 02:22:38 +0100
Subject: [PATCH 21/31] fix: need to set the null byte here

---
 src/vm/datatypes/strings.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 3451137a6..e569772b1 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -431,8 +431,8 @@ static Value lowerString(DictuVM *vm, int argCount, Value *args) {
     }
     char *temp = ALLOCATE(vm, char, string->length + 1);
     memcpy(temp, string->chars, string->length);
-    utf8lwr(temp);
     temp[string->length] = '\0';
+    utf8lwr(temp);
 
     return OBJ_VAL(takeStringWithLen(vm, temp, string->length, string->character_len));
 }

From e8ceb820e9fcf07bda85542fe53aab86fd7b62f2 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Wed, 6 Nov 2024 02:24:59 +0100
Subject: [PATCH 22/31] remove unused var

---
 src/vm/datatypes/strings.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index e569772b1..77d966fe1 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -94,7 +94,6 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
     }
 
 
-    int stringLen = string->length + 1;
     char *tmp = string->chars;
     char *tmpFree = tmp;
 
@@ -113,7 +112,6 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
             free(replaceStrings[i]);
         }
 
-        FREE_ARRAY(vm, char, tmp , stringLen);
         FREE_ARRAY(vm, char*, replaceStrings, argCount);
         return EMPTY_VAL;
     }
@@ -451,7 +449,7 @@ static Value upperString(DictuVM *vm, int argCount, Value *args) {
 
     char *temp = ALLOCATE(vm, char, string->length + 1);
     memcpy(temp, string->chars, string->length);
-    temp[string->length] = 0x00;
+    temp[string->length] = '\0';
     utf8upr(temp);
 
     return OBJ_VAL(takeStringWithLen(vm, temp, string->length, string->character_len));

From f8d481843ba1c1529f572b8dc378f58dbd17b053 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Thu, 7 Nov 2024 02:00:47 +0100
Subject: [PATCH 23/31] fix issue with uuid module

---
 src/optionals/uuid.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/optionals/uuid.c b/src/optionals/uuid.c
index 6f5be59f1..28d5e324d 100644
--- a/src/optionals/uuid.c
+++ b/src/optionals/uuid.c
@@ -3,7 +3,9 @@
 #ifndef _WIN32
 static uuid_t uuid;
 
-#define UUID_STRING_LEN 36
+#define UUID_STRING_LEN 36 // Note(Liz3): the strlen() of a UUID is 36, but the uuid_unparse_lower call expects
+                           // a buffer with space for the null byte, but since copyString wants the strlen we
+                           // keep the define to 36 and allocate +1.
 
 static Value uuidGenerateNative(DictuVM *vm, int argCount, Value *args) {
     UNUSED(args);
@@ -21,7 +23,7 @@ static Value uuidGenerateNative(DictuVM *vm, int argCount, Value *args) {
         return newResultError(vm, "error: failed to generate uuid");
     }
 
-    char out[UUID_STRING_LEN] = {};
+    char out[UUID_STRING_LEN+1] = {};
     uuid_unparse_lower(uuid, out);
 
     return newResultSuccess(vm, OBJ_VAL(copyString(vm, out, UUID_STRING_LEN)));
@@ -43,7 +45,7 @@ static Value uuidGenRandomNative(DictuVM *vm, int argCount, Value *args) {
         return newResultError(vm, "error: failed to generate uuid");
     }
 
-    char out[UUID_STRING_LEN] = {};
+    char out[UUID_STRING_LEN+1] = {};
     uuid_unparse_lower(uuid, out);
 
     return newResultSuccess(vm, OBJ_VAL(copyString(vm, out, UUID_STRING_LEN)));
@@ -65,7 +67,7 @@ static Value uuidGenTimeNative(DictuVM *vm, int argCount, Value *args) {
         return newResultError(vm, "error: failed to generate uuid");
     }
 
-    char out[UUID_STRING_LEN] = {};
+    char out[UUID_STRING_LEN+1] = {};
     uuid_unparse_lower(uuid, out);
 
     return newResultSuccess(vm, OBJ_VAL(copyString(vm, out, UUID_STRING_LEN)));

From 9d89a15af0d7e5c76f3fb2e084e3f76d568b2062 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Thu, 7 Nov 2024 04:37:54 +0100
Subject: [PATCH 24/31] chore: some reformatting

---
 src/vm/datatypes/strings.c | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 77d966fe1..c568c02d2 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -702,17 +702,14 @@ static Value isUpperString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "isUpper() takes no arguments (%d given)", argCount);
         return EMPTY_VAL;
     }
-    {
-        ObjString *string = AS_STRING(args[0]);
-        if(string->character_len == -1) {
+
+    ObjString* strObj = AS_STRING(args[0]);
+    if(strObj->character_len == -1) {
            runtimeError(vm, "String contains invalid UTF-8", argCount);
            return EMPTY_VAL;
-       }
     }
-
-
-    char *string = AS_CSTRING(args[0]);
-    int len = utf8len(string);
+    char *string = strObj->chars;
+    int len = strObj->character_len;
 
     if (len == 0) {
         return BOOL_VAL(false);
@@ -734,16 +731,14 @@ static Value isLowerString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "isLower() takes no arguments (%d given)", argCount);
         return EMPTY_VAL;
     }
-    {
-        ObjString *string = AS_STRING(args[0]);
-        if(string->character_len == -1) {
-           runtimeError(vm, "String contains invalid UTF-8", argCount);
-           return EMPTY_VAL;
-       }
+    ObjString *strObj = AS_STRING(args[0]);
+    if(strObj->character_len == -1) {
+       runtimeError(vm, "String contains invalid UTF-8", argCount);
+       return EMPTY_VAL;
     }
 
-    char *string = AS_CSTRING(args[0]);
-    int len = utf8len(string);
+    char *string = strObj->chars;
+    int len = strObj->character_len;
 
     if (len == 0) {
         return BOOL_VAL(false);
@@ -793,7 +788,7 @@ static Value collapseSpacesString(DictuVM *vm, int argCount, Value *args) {
     if (targetLen != string->length) {
         dest = SHRINK_ARRAY(vm, dest, char, string->length + 1, targetLen + 1);
     }
-    dest[targetLen] = 0x00;
+    dest[targetLen] = '\0';
 
     return OBJ_VAL(takeString(vm, dest, targetLen));
 }
@@ -810,7 +805,7 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
        return EMPTY_VAL;
    }
     char *temp = ALLOCATE(vm, char, string->length + 1);
-    temp[string->length] = 0x00;
+    temp[string->length] = '\0';
     memcpy(temp, string->chars, string->length);
     int len = AS_NUMBER(args[1]);
 

From 44bc1ec1d155b993a06c47afd769430224395541 Mon Sep 17 00:00:00 2001
From: Liz <accs@liz3.net>
Date: Fri, 8 Nov 2024 00:45:18 +0100
Subject: [PATCH 25/31] Update src/vm/datatypes/strings.c

Co-authored-by: Jason_000 <Jason2605@users.noreply.github.com>
---
 src/vm/datatypes/strings.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index c568c02d2..4b1cfab71 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -815,7 +815,6 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
     for (int cur = 0; cur < string->character_len; cur++, count++) {
         utf8_int32_t cp;
         char* n = utf8codepoint(ptr, &cp);
-        // temp[cur] = string->chars[cur];
 
         if (isspace(cp)) {
             last = ptr - temp;

From aaac64e12e09747bf120bcfe813509a69d833849 Mon Sep 17 00:00:00 2001
From: Liz <accs@liz3.net>
Date: Fri, 8 Nov 2024 00:45:31 +0100
Subject: [PATCH 26/31] Update src/vm/datatypes/strings.c

Co-authored-by: Jason_000 <Jason2605@users.noreply.github.com>
---
 src/vm/datatypes/strings.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 4b1cfab71..2c6ef5e69 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -346,7 +346,7 @@ static Value findLastString(DictuVM *vm, int argCount, Value *args) {
         ptr = utf8rcodepoint(ptr, &cp);
     }
 
-    return NUMBER_VAL( -1);
+    return NUMBER_VAL(-1);
 }
 
 static Value replaceString(DictuVM *vm, int argCount, Value *args) {

From 21e3d17aea833456880fd2e4f679f8d449ae220e Mon Sep 17 00:00:00 2001
From: Liz <accs@liz3.net>
Date: Fri, 8 Nov 2024 00:45:44 +0100
Subject: [PATCH 27/31] Update src/vm/datatypes/strings.c

Co-authored-by: Jason_000 <Jason2605@users.noreply.github.com>
---
 src/vm/datatypes/strings.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 2c6ef5e69..85c6a1ea1 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -803,7 +803,7 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
     if(string->character_len == -1) {
        runtimeError(vm, "String contains invalid UTF-8", argCount);
        return EMPTY_VAL;
-   }
+    }
     char *temp = ALLOCATE(vm, char, string->length + 1);
     temp[string->length] = '\0';
     memcpy(temp, string->chars, string->length);

From 522d4324118ad23d5b12f517f7d5bdc87ed3b49b Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Fri, 8 Nov 2024 01:12:30 +0100
Subject: [PATCH 28/31] refactor scopes

---
 src/vm/datatypes/strings.c | 152 +++++++++++++++++++++++--------------
 1 file changed, 96 insertions(+), 56 deletions(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 85c6a1ea1..b53f4023d 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -153,11 +153,16 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     runtimeError(vm, "String contains invalid UTF-8");
      return EMPTY_VAL;
      }
 
-    char *delimiter = AS_CSTRING(args[1]);
+    ObjString* delimiterObj = AS_STRING(args[1]);
+    if(delimiterObj->character_len == -1) {
+       runtimeError(vm, "Delimiter String contains invalid UTF-8");
+       return EMPTY_VAL;
+    }
+    char *delimiter = delimiterObj->chars;
     int maxSplit = string->length + 1;
 
     if (argCount == 2) {
@@ -243,20 +248,26 @@ static Value containsString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "contains() takes 1 argument (%d given)", argCount);
         return EMPTY_VAL;
     }
-    {
-        ObjString *string = AS_STRING(args[0]);
-        if(string->character_len == -1) {
-           runtimeError(vm, "String contains invalid UTF-8", argCount);
-           return EMPTY_VAL;
-       }
-    }
+    
+    ObjString *strObj = AS_STRING(args[0]);
+    if(strObj->character_len == -1) {
+       runtimeError(vm, "String contains invalid UTF-8");
+       return EMPTY_VAL;
+   }
+    
     if (!IS_STRING(args[1])) {
         runtimeError(vm, "Argument passed to contains() must be a string");
         return EMPTY_VAL;
-    }
+    } 
+    ObjString *delimiterObj = AS_STRING(args[1]);
+    if(delimiterObj->character_len == -1) {
+       runtimeError(vm, "Search String contains invalid UTF-8");
+       return EMPTY_VAL;
+   }
+    
 
-    char *string = AS_CSTRING(args[0]);
-    char *delimiter = AS_CSTRING(args[1]);
+    char *string = strObj->chars;
+    char *delimiter = delimiterObj->chars;
 
     if (!utf8str(string, delimiter)) {
         return FALSE_VAL;
@@ -270,13 +281,13 @@ static Value findString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "find() takes either 1 or 2 arguments (%d given)", argCount);
         return EMPTY_VAL;
     }
-    {
-        ObjString *string = AS_STRING(args[0]);
-        if(string->character_len == -1) {
-           runtimeError(vm, "String contains invalid UTF-8", argCount);
-           return EMPTY_VAL;
-       }
-    }
+    
+    ObjString *strObj = AS_STRING(args[0]);
+    if(strObj->character_len == -1) {
+       runtimeError(vm, "String contains invalid UTF-8");
+       return EMPTY_VAL;
+   }
+    
     int index = 1;
 
     if (argCount == 2) {
@@ -291,10 +302,16 @@ static Value findString(DictuVM *vm, int argCount, Value *args) {
     if (!IS_STRING(args[1])) {
         runtimeError(vm, "Substring passed to find() must be a string");
         return EMPTY_VAL;
+    } 
+    ObjString *substrObj = AS_STRING(args[1]);
+    if(substrObj->character_len == -1) {
+       runtimeError(vm, "Substring String contains invalid UTF-8");
+       return EMPTY_VAL;
     }
+    
 
-    char *substr = AS_CSTRING(args[1]);
-    char *string = AS_CSTRING(args[0]);
+    char *string = strObj->chars;
+    char *substr = substrObj->chars;
 
     int position = 0;
 
@@ -319,16 +336,21 @@ static Value findLastString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "findLast() takes 1 argument (%d given)", argCount);
         return EMPTY_VAL;
     }
-    {
-        ObjString *string = AS_STRING(args[0]);
-        if(string->character_len == -1) {
-           runtimeError(vm, "String contains invalid UTF-8", argCount);
-           return EMPTY_VAL;
-       }
+    if (!IS_STRING(args[1])) {
+        runtimeError(vm, "Substring passed to findLast() must be a string");
+        return EMPTY_VAL;
     }
     utf8_int32_t cp;
     ObjString *str = AS_STRING(args[0]);
     ObjString *ss = AS_STRING(args[1]);
+    if(str->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
+    }
+    if(ss->character_len == -1) {
+       runtimeError(vm, "Search String contains invalid UTF-8");
+       return EMPTY_VAL;
+    }
     if(ss->length > str->length || ss->character_len > str->character_len)
         return NUMBER_VAL(-1);
 
@@ -359,19 +381,25 @@ static Value replaceString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "Arguments passed to replace() must be a strings");
         return EMPTY_VAL;
     }
-    {
-        ObjString *string = AS_STRING(args[0]);
-        if(string->character_len == -1) {
-           runtimeError(vm, "String contains invalid UTF-8", argCount);
-           return EMPTY_VAL;
-       }
-    }
-
     // Pop values off the stack
-    Value stringValue = args[0];
+    ObjString *stringValue = AS_STRING(args[0]);
+    if(stringValue->character_len == -1) {
+       runtimeError(vm, "String contains invalid UTF-8");
+       return EMPTY_VAL;
+   }
+    
+
     ObjString *to_replace = AS_STRING(args[1]);
     ObjString *replace = AS_STRING(args[2]);
-    char *string = AS_CSTRING(stringValue);
+    if(to_replace->character_len == -1) {
+       runtimeError(vm, "To Replace String contains invalid UTF-8");
+       return EMPTY_VAL;
+    }
+    if(replace->character_len == -1) {
+       runtimeError(vm, "Replace String contains invalid UTF-8");
+       return EMPTY_VAL;
+    }
+    char *string = stringValue->chars;
 
     int count = 0;
     int len = to_replace->length;
@@ -392,7 +420,7 @@ static Value replaceString(DictuVM *vm, int argCount, Value *args) {
     tmp = tmpFree;
 
     if (count == 0) {
-        return stringValue;
+        return OBJ_VAL(stringValue);
     }
 
     int length = utf8size_lazy(tmp) - count * (len - replaceLen) + 1;
@@ -424,7 +452,7 @@ static Value lowerString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     runtimeError(vm, "String contains invalid UTF-8");
      return EMPTY_VAL;
     }
     char *temp = ALLOCATE(vm, char, string->length + 1);
@@ -443,7 +471,7 @@ static Value upperString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     runtimeError(vm, "String contains invalid UTF-8");
      return EMPTY_VAL;
     }
 
@@ -468,11 +496,15 @@ static Value startsWithString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     runtimeError(vm, "String contains invalid UTF-8");
      return EMPTY_VAL;
     }
 
     ObjString *start = AS_STRING(args[1]);
+    if(start->character_len == -1) {
+       runtimeError(vm, "Prefix String contains invalid UTF-8");
+       return EMPTY_VAL;
+    }
     if(string->length < start->length)
         return BOOL_VAL(false);
 
@@ -492,11 +524,15 @@ static Value endsWithString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     runtimeError(vm, "String contains invalid UTF-8");
      return EMPTY_VAL;
     }
 
     ObjString *suffix = AS_STRING(args[1]);
+    if(suffix->character_len == -1) {
+       runtimeError(vm, "Suffix String contains invalid UTF-8");
+       return EMPTY_VAL;
+    }
 
     if (string->length < suffix->length) {
         return FALSE_VAL;
@@ -580,16 +616,20 @@ static Value countString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "Argument passed to count() must be a string");
         return EMPTY_VAL;
     }
-    {
-        ObjString *string = AS_STRING(args[0]);
-        if(string->character_len == -1) {
-           runtimeError(vm, "String contains invalid UTF-8", argCount);
-           return EMPTY_VAL;
-       }
+    
+    ObjString *string = AS_STRING(args[0]);
+    ObjString *needleObj = AS_STRING(args[1]);
+    if(string->character_len == -1) {
+       runtimeError(vm, "String contains invalid UTF-8");
+       return EMPTY_VAL;
+    }
+    if(needleObj->character_len == -1) {
+       runtimeError(vm, "Needle contains invalid UTF-8");
+       return EMPTY_VAL;
     }
 
-    char *haystack = AS_CSTRING(args[0]);
-    char *needle = AS_CSTRING(args[1]);
+    char *haystack = string->chars;
+    char *needle = needleObj->chars;
 
     int count = 0;
     while ((haystack = utf8str(haystack, needle))) {
@@ -609,7 +649,7 @@ static Value wordCountString(DictuVM *vm, int argCount, Value *args) {
     ObjString *string = AS_STRING(args[0]);
     
     if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     runtimeError(vm, "String contains invalid UTF-8");
      return EMPTY_VAL;
     }
     int count = 0;
@@ -641,7 +681,7 @@ static Value titleString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8", argCount);
+     runtimeError(vm, "String contains invalid UTF-8");
      return EMPTY_VAL;
     }
 
@@ -705,7 +745,7 @@ static Value isUpperString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString* strObj = AS_STRING(args[0]);
     if(strObj->character_len == -1) {
-           runtimeError(vm, "String contains invalid UTF-8", argCount);
+           runtimeError(vm, "String contains invalid UTF-8");
            return EMPTY_VAL;
     }
     char *string = strObj->chars;
@@ -733,7 +773,7 @@ static Value isLowerString(DictuVM *vm, int argCount, Value *args) {
     }
     ObjString *strObj = AS_STRING(args[0]);
     if(strObj->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8", argCount);
+       runtimeError(vm, "String contains invalid UTF-8");
        return EMPTY_VAL;
     }
 
@@ -764,7 +804,7 @@ static Value collapseSpacesString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     if(string->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8", argCount);
+       runtimeError(vm, "String contains invalid UTF-8");
        return EMPTY_VAL;
    }
     char *dest = ALLOCATE(vm, char, string->length + 1);
@@ -801,7 +841,7 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
 
     ObjString *string = AS_STRING(args[0]);
     if(string->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8", argCount);
+       runtimeError(vm, "String contains invalid UTF-8");
        return EMPTY_VAL;
     }
     char *temp = ALLOCATE(vm, char, string->length + 1);

From d0bf48b68d5ba2bdd93a54de78fec8cdd2090a9d Mon Sep 17 00:00:00 2001
From: Liz <accs@liz3.net>
Date: Fri, 8 Nov 2024 01:13:29 +0100
Subject: [PATCH 29/31] Update src/vm/datatypes/strings.c

Co-authored-by: Jason_000 <Jason2605@users.noreply.github.com>
---
 src/vm/datatypes/strings.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index b53f4023d..bed082c75 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -24,7 +24,7 @@ static Value lenString(DictuVM *vm, int argCount, Value *args) {
 
 static Value byteLenString(DictuVM *vm, int argCount, Value *args) {
     if (argCount != 0) {
-        runtimeError(vm, "len() takes no arguments (%d given)", argCount);
+        runtimeError(vm, "byteLen() takes no arguments (%d given)", argCount);
         return EMPTY_VAL;
     }
 

From c0f35335249f9b66dee10409a94ff65207bfdb11 Mon Sep 17 00:00:00 2001
From: Liz <accs@liz3.net>
Date: Fri, 8 Nov 2024 01:14:22 +0100
Subject: [PATCH 30/31] Update src/vm/datatypes/strings.c

Co-authored-by: Jason_000 <Jason2605@users.noreply.github.com>
---
 src/vm/datatypes/strings.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index bed082c75..7b3b3e39e 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -192,8 +192,6 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
           
             char* next_ptr = utf8codepoint(ptr, &cp);
 
-            // *(tmp) = string->chars[tokenIndex];
-            // *(tmp + 1) = '\0';
             size_t size = next_ptr - ptr;
             Value str = OBJ_VAL(copyString(vm, ptr, size));
             // Push to stack to avoid GC

From 9fa46e34ec71e0f90d82d4a98eadc28c7cc036f6 Mon Sep 17 00:00:00 2001
From: Liz3 <accs@liz3.net>
Date: Fri, 8 Nov 2024 01:17:41 +0100
Subject: [PATCH 31/31] style: format file

---
 src/vm/datatypes/strings.c | 325 +++++++++++++++++++------------------
 1 file changed, 168 insertions(+), 157 deletions(-)

diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c
index 7b3b3e39e..d42440681 100644
--- a/src/vm/datatypes/strings.c
+++ b/src/vm/datatypes/strings.c
@@ -15,8 +15,9 @@ static Value lenString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    if(string->character_len == -1) {
-        runtimeError(vm, "String contains invalid UTF-8, consider using byteLen()");
+    if (string->character_len == -1) {
+        runtimeError(vm,
+                     "String contains invalid UTF-8, consider using byteLen()");
         return EMPTY_VAL;
     }
     return NUMBER_VAL(string->character_len);
@@ -66,18 +67,19 @@ static Value toNumberString(DictuVM *vm, int argCount, Value *args) {
 
 static Value formatString(DictuVM *vm, int argCount, Value *args) {
     if (argCount == 0) {
-        runtimeError(vm, "format() takes at least 1 argument (%d given)", argCount);
+        runtimeError(vm, "format() takes at least 1 argument (%d given)",
+                     argCount);
         return EMPTY_VAL;
     }
     ObjString *string = AS_STRING(args[0]);
 
-    if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8", argCount);
-     return EMPTY_VAL;
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8", argCount);
+        return EMPTY_VAL;
     }
 
     int length = 0;
-    char **replaceStrings = ALLOCATE(vm, char*, argCount);
+    char **replaceStrings = ALLOCATE(vm, char *, argCount);
 
     for (int j = 1; j < argCount + 1; j++) {
         Value value = args[j];
@@ -93,7 +95,6 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
         length += utf8size_lazy(replaceStrings[j - 1]);
     }
 
-
     char *tmp = string->chars;
     char *tmpFree = tmp;
 
@@ -112,7 +113,7 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
             free(replaceStrings[i]);
         }
 
-        FREE_ARRAY(vm, char*, replaceStrings, argCount);
+        FREE_ARRAY(vm, char *, replaceStrings, argCount);
         return EMPTY_VAL;
     }
 
@@ -127,13 +128,14 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) {
         int tmpLength = pos - tmp;
         int replaceLength = utf8size_lazy(replaceStrings[i]);
         memcpy(newStr + stringLength, tmp, tmpLength);
-        memcpy(newStr + stringLength + tmpLength, replaceStrings[i], replaceLength);
+        memcpy(newStr + stringLength + tmpLength, replaceStrings[i],
+               replaceLength);
         stringLength += tmpLength + replaceLength;
         tmp = pos + 2;
         free(replaceStrings[i]);
     }
 
-    FREE_ARRAY(vm, char*, replaceStrings, argCount);
+    FREE_ARRAY(vm, char *, replaceStrings, argCount);
     memcpy(newStr + stringLength, tmp, utf8size_lazy(tmp));
     newStr[fullLength - 1] = '\0';
 
@@ -152,15 +154,15 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8");
-     return EMPTY_VAL;
-     }
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
+    }
 
-    ObjString* delimiterObj = AS_STRING(args[1]);
-    if(delimiterObj->character_len == -1) {
-       runtimeError(vm, "Delimiter String contains invalid UTF-8");
-       return EMPTY_VAL;
+    ObjString *delimiterObj = AS_STRING(args[1]);
+    if (delimiterObj->character_len == -1) {
+        runtimeError(vm, "Delimiter String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
     char *delimiter = delimiterObj->chars;
     int maxSplit = string->length + 1;
@@ -183,14 +185,15 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
     ObjList *list = newList(vm);
     push(vm, OBJ_VAL(list));
     int count = 0;
-    char* ptr = string->chars;
+    char *ptr = string->chars;
     utf8_int32_t cp;
     if (delimiterLength == 0) {
         int tokenIndex = 0;
-        for (; tokenIndex < string->character_len && count < maxSplit; tokenIndex++) {
+        for (; tokenIndex < string->character_len && count < maxSplit;
+             tokenIndex++) {
             count++;
-          
-            char* next_ptr = utf8codepoint(ptr, &cp);
+
+            char *next_ptr = utf8codepoint(ptr, &cp);
 
             size_t size = next_ptr - ptr;
             Value str = OBJ_VAL(copyString(vm, ptr, size));
@@ -211,7 +214,9 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) {
             count++;
             token = utf8str(tmp, delimiter);
 
-            Value str = OBJ_VAL(copyString(vm, tmp, token == NULL ? utf8size_lazy(tmp) : ((size_t)(token-tmp))));
+            Value str = OBJ_VAL(copyString(
+                vm, tmp,
+                token == NULL ? utf8size_lazy(tmp) : ((size_t)(token - tmp))));
 
             // Push to stack to avoid GC
             push(vm, str);
@@ -246,23 +251,22 @@ static Value containsString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "contains() takes 1 argument (%d given)", argCount);
         return EMPTY_VAL;
     }
-    
+
     ObjString *strObj = AS_STRING(args[0]);
-    if(strObj->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8");
-       return EMPTY_VAL;
-   }
-    
+    if (strObj->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
+    }
+
     if (!IS_STRING(args[1])) {
         runtimeError(vm, "Argument passed to contains() must be a string");
         return EMPTY_VAL;
-    } 
+    }
     ObjString *delimiterObj = AS_STRING(args[1]);
-    if(delimiterObj->character_len == -1) {
-       runtimeError(vm, "Search String contains invalid UTF-8");
-       return EMPTY_VAL;
-   }
-    
+    if (delimiterObj->character_len == -1) {
+        runtimeError(vm, "Search String contains invalid UTF-8");
+        return EMPTY_VAL;
+    }
 
     char *string = strObj->chars;
     char *delimiter = delimiterObj->chars;
@@ -276,16 +280,17 @@ static Value containsString(DictuVM *vm, int argCount, Value *args) {
 
 static Value findString(DictuVM *vm, int argCount, Value *args) {
     if (argCount < 1 || argCount > 2) {
-        runtimeError(vm, "find() takes either 1 or 2 arguments (%d given)", argCount);
+        runtimeError(vm, "find() takes either 1 or 2 arguments (%d given)",
+                     argCount);
         return EMPTY_VAL;
     }
-    
+
     ObjString *strObj = AS_STRING(args[0]);
-    if(strObj->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8");
-       return EMPTY_VAL;
-   }
-    
+    if (strObj->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
+    }
+
     int index = 1;
 
     if (argCount == 2) {
@@ -300,13 +305,12 @@ static Value findString(DictuVM *vm, int argCount, Value *args) {
     if (!IS_STRING(args[1])) {
         runtimeError(vm, "Substring passed to find() must be a string");
         return EMPTY_VAL;
-    } 
+    }
     ObjString *substrObj = AS_STRING(args[1]);
-    if(substrObj->character_len == -1) {
-       runtimeError(vm, "Substring String contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (substrObj->character_len == -1) {
+        runtimeError(vm, "Substring String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
-    
 
     char *string = strObj->chars;
     char *substr = substrObj->chars;
@@ -314,15 +318,15 @@ static Value findString(DictuVM *vm, int argCount, Value *args) {
     int position = 0;
 
     for (int i = 0; i < index; ++i) {
-        if(i != 0) {
-            position +=  utf8len(substr);
+        if (i != 0) {
+            position += utf8len(substr);
         }
         char *result = utf8str(string, substr);
         if (!result) {
             position = -1;
             break;
         }
-        position += utf8nlen(string, (result-string));
+        position += utf8nlen(string, (result - string));
         string = result + utf8size_lazy(substr);
     }
 
@@ -341,27 +345,27 @@ static Value findLastString(DictuVM *vm, int argCount, Value *args) {
     utf8_int32_t cp;
     ObjString *str = AS_STRING(args[0]);
     ObjString *ss = AS_STRING(args[1]);
-    if(str->character_len == -1) {
+    if (str->character_len == -1) {
         runtimeError(vm, "String contains invalid UTF-8");
         return EMPTY_VAL;
     }
-    if(ss->character_len == -1) {
-       runtimeError(vm, "Search String contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (ss->character_len == -1) {
+        runtimeError(vm, "Search String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
-    if(ss->length > str->length || ss->character_len > str->character_len)
+    if (ss->length > str->length || ss->character_len > str->character_len)
         return NUMBER_VAL(-1);
 
-    char* ptr = str->chars;
+    char *ptr = str->chars;
     int index = str->character_len - ss->character_len;
-    for(int i = 0; i< index; i++)
-          ptr = utf8codepoint(ptr, &cp);
+    for (int i = 0; i < index; i++)
+        ptr = utf8codepoint(ptr, &cp);
 
-    while(true) {
-        if(utf8str(ptr, ss->chars) == ptr)
+    while (true) {
+        if (utf8str(ptr, ss->chars) == ptr)
             return NUMBER_VAL(index);
         index--;
-        if(index < 0)
+        if (index < 0)
             break;
         ptr = utf8rcodepoint(ptr, &cp);
     }
@@ -381,21 +385,20 @@ static Value replaceString(DictuVM *vm, int argCount, Value *args) {
     }
     // Pop values off the stack
     ObjString *stringValue = AS_STRING(args[0]);
-    if(stringValue->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8");
-       return EMPTY_VAL;
-   }
-    
+    if (stringValue->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
+    }
 
     ObjString *to_replace = AS_STRING(args[1]);
     ObjString *replace = AS_STRING(args[2]);
-    if(to_replace->character_len == -1) {
-       runtimeError(vm, "To Replace String contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (to_replace->character_len == -1) {
+        runtimeError(vm, "To Replace String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
-    if(replace->character_len == -1) {
-       runtimeError(vm, "Replace String contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (replace->character_len == -1) {
+        runtimeError(vm, "Replace String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
     char *string = stringValue->chars;
 
@@ -449,16 +452,17 @@ static Value lowerString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8");
-     return EMPTY_VAL;
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
     char *temp = ALLOCATE(vm, char, string->length + 1);
     memcpy(temp, string->chars, string->length);
     temp[string->length] = '\0';
     utf8lwr(temp);
 
-    return OBJ_VAL(takeStringWithLen(vm, temp, string->length, string->character_len));
+    return OBJ_VAL(
+        takeStringWithLen(vm, temp, string->length, string->character_len));
 }
 
 static Value upperString(DictuVM *vm, int argCount, Value *args) {
@@ -468,9 +472,9 @@ static Value upperString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8");
-     return EMPTY_VAL;
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
 
     char *temp = ALLOCATE(vm, char, string->length + 1);
@@ -478,7 +482,8 @@ static Value upperString(DictuVM *vm, int argCount, Value *args) {
     temp[string->length] = '\0';
     utf8upr(temp);
 
-    return OBJ_VAL(takeStringWithLen(vm, temp, string->length, string->character_len));
+    return OBJ_VAL(
+        takeStringWithLen(vm, temp, string->length, string->character_len));
 }
 
 static Value startsWithString(DictuVM *vm, int argCount, Value *args) {
@@ -493,17 +498,17 @@ static Value startsWithString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8");
-     return EMPTY_VAL;
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
 
     ObjString *start = AS_STRING(args[1]);
-    if(start->character_len == -1) {
-       runtimeError(vm, "Prefix String contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (start->character_len == -1) {
+        runtimeError(vm, "Prefix String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
-    if(string->length < start->length)
+    if (string->length < start->length)
         return BOOL_VAL(false);
 
     return BOOL_VAL(utf8ncmp(string->chars, start->chars, start->length) == 0);
@@ -521,22 +526,23 @@ static Value endsWithString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8");
-     return EMPTY_VAL;
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
 
     ObjString *suffix = AS_STRING(args[1]);
-    if(suffix->character_len == -1) {
-       runtimeError(vm, "Suffix String contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (suffix->character_len == -1) {
+        runtimeError(vm, "Suffix String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
 
     if (string->length < suffix->length) {
         return FALSE_VAL;
     }
 
-    return BOOL_VAL(utf8cmp(string->chars + (string->length - suffix->length), suffix->chars) == 0);
+    return BOOL_VAL(utf8cmp(string->chars + (string->length - suffix->length),
+                            suffix->chars) == 0);
 }
 
 static Value leftStripString(DictuVM *vm, int argCount, Value *args) {
@@ -557,7 +563,8 @@ static Value leftStripString(DictuVM *vm, int argCount, Value *args) {
     }
 
     if (count != 0) {
-        temp = SHRINK_ARRAY(vm, temp, char, string->length + 1, (string->length - count) + 1);
+        temp = SHRINK_ARRAY(vm, temp, char, string->length + 1,
+                            (string->length - count) + 1);
     }
 
     memcpy(temp, string->chars + count, string->length - count);
@@ -567,7 +574,8 @@ static Value leftStripString(DictuVM *vm, int argCount, Value *args) {
 
 static Value rightStripString(DictuVM *vm, int argCount, Value *args) {
     if (argCount != 0) {
-        runtimeError(vm, "rightStrip() takes no arguments (%d given)", argCount);
+        runtimeError(vm, "rightStrip() takes no arguments (%d given)",
+                     argCount);
         return EMPTY_VAL;
     }
 
@@ -614,16 +622,16 @@ static Value countString(DictuVM *vm, int argCount, Value *args) {
         runtimeError(vm, "Argument passed to count() must be a string");
         return EMPTY_VAL;
     }
-    
+
     ObjString *string = AS_STRING(args[0]);
     ObjString *needleObj = AS_STRING(args[1]);
-    if(string->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
-    if(needleObj->character_len == -1) {
-       runtimeError(vm, "Needle contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (needleObj->character_len == -1) {
+        runtimeError(vm, "Needle contains invalid UTF-8");
+        return EMPTY_VAL;
     }
 
     char *haystack = string->chars;
@@ -645,23 +653,23 @@ static Value wordCountString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    
-    if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8");
-     return EMPTY_VAL;
+
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
     int count = 0;
     int len = string->character_len;
     bool in = false;
-    char* ptr = string->chars;
+    char *ptr = string->chars;
 
     for (int i = 0; i < len; i++) {
         utf8_int32_t cp;
         ptr = utf8codepoint(ptr, &cp);
         if (iswspace(cp)) {
             in = false;
-        } else if(iswalpha(cp)) {
-            if(!in) {
+        } else if (iswalpha(cp)) {
+            if (!in) {
                 in = true;
                 count++;
             }
@@ -678,36 +686,36 @@ static Value titleString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    if(string->character_len == -1) {
-     runtimeError(vm, "String contains invalid UTF-8");
-     return EMPTY_VAL;
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
 
     char *temp = ALLOCATE(vm, char, string->length + 1);
     memcpy(temp, string->chars, string->length);
-    char* ptr = temp;
-    bool convertNext=true;
+    char *ptr = temp;
+    bool convertNext = true;
 
     for (int i = 0; i < string->character_len; i++) {
         utf8_int32_t cp;
-        char* next_ptr = utf8codepoint(ptr, &cp);
-        if(cp==' '){
-            convertNext=true;
-        }
-        else if(convertNext){
+        char *next_ptr = utf8codepoint(ptr, &cp);
+        if (cp == ' ') {
+            convertNext = true;
+        } else if (convertNext) {
             cp = utf8uprcodepoint(cp);
-            convertNext=false;
-          
+            convertNext = false;
+
         } else {
             cp = utf8lwrcodepoint(cp);
         }
-        utf8catcodepoint(ptr, cp, next_ptr -ptr);
+        utf8catcodepoint(ptr, cp, next_ptr - ptr);
         ptr = next_ptr;
     }
 
     temp[string->length] = '\0';
 
-    return OBJ_VAL(takeStringWithLen(vm, temp, string->length, string->character_len));
+    return OBJ_VAL(
+        takeStringWithLen(vm, temp, string->length, string->character_len));
 }
 
 static Value repeatString(DictuVM *vm, int argCount, Value *args) {
@@ -741,10 +749,10 @@ static Value isUpperString(DictuVM *vm, int argCount, Value *args) {
         return EMPTY_VAL;
     }
 
-    ObjString* strObj = AS_STRING(args[0]);
-    if(strObj->character_len == -1) {
-           runtimeError(vm, "String contains invalid UTF-8");
-           return EMPTY_VAL;
+    ObjString *strObj = AS_STRING(args[0]);
+    if (strObj->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
     char *string = strObj->chars;
     int len = strObj->character_len;
@@ -760,7 +768,7 @@ static Value isUpperString(DictuVM *vm, int argCount, Value *args) {
             return BOOL_VAL(false);
         }
     }
-    
+
     return BOOL_VAL(true);
 }
 
@@ -770,9 +778,9 @@ static Value isLowerString(DictuVM *vm, int argCount, Value *args) {
         return EMPTY_VAL;
     }
     ObjString *strObj = AS_STRING(args[0]);
-    if(strObj->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (strObj->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
 
     char *string = strObj->chars;
@@ -783,39 +791,40 @@ static Value isLowerString(DictuVM *vm, int argCount, Value *args) {
     }
 
     for (int i = 0; i < len; i++) {
-         utf8_int32_t cp;
+        utf8_int32_t cp;
         string = utf8codepoint(string, &cp);
-     
+
         if (!iswlower(cp) && iswalpha(cp)) {
             return BOOL_VAL(false);
         }
     }
-    
+
     return BOOL_VAL(true);
 }
 
 static Value collapseSpacesString(DictuVM *vm, int argCount, Value *args) {
     if (argCount != 0) {
-        runtimeError(vm, "collapseSpaces() takes no arguments (%d given)", argCount);
+        runtimeError(vm, "collapseSpaces() takes no arguments (%d given)",
+                     argCount);
         return EMPTY_VAL;
     }
 
     ObjString *string = AS_STRING(args[0]);
-    if(string->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8");
-       return EMPTY_VAL;
-   }
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
+    }
     char *dest = ALLOCATE(vm, char, string->length + 1);
-    char* ptr = string->chars;
+    char *ptr = string->chars;
     int targetLen = 0;
     int i, j;
     utf8_int32_t last;
     for (i = j = 0; i < string->character_len; ++i) {
         utf8_int32_t cp;
-        char* n = utf8codepoint(ptr, &cp);
-        
+        char *n = utf8codepoint(ptr, &cp);
+
         if (!iswspace(cp) || (i > 0 && !iswspace(last))) {
-            utf8catcodepoint(dest+targetLen, cp, string->length - targetLen);
+            utf8catcodepoint(dest + targetLen, cp, string->length - targetLen);
             targetLen += utf8codepointsize(cp);
         }
 
@@ -838,9 +847,9 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
     }
 
     ObjString *string = AS_STRING(args[0]);
-    if(string->character_len == -1) {
-       runtimeError(vm, "String contains invalid UTF-8");
-       return EMPTY_VAL;
+    if (string->character_len == -1) {
+        runtimeError(vm, "String contains invalid UTF-8");
+        return EMPTY_VAL;
     }
     char *temp = ALLOCATE(vm, char, string->length + 1);
     temp[string->length] = '\0';
@@ -849,14 +858,14 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
 
     int last = -1;
     int count = 0;
-    char*ptr = temp;
+    char *ptr = temp;
     for (int cur = 0; cur < string->character_len; cur++, count++) {
         utf8_int32_t cp;
-        char* n = utf8codepoint(ptr, &cp);
+        char *n = utf8codepoint(ptr, &cp);
 
         if (isspace(cp)) {
             last = ptr - temp;
-        } 
+        }
 
         if (count >= len && last != -1) {
             temp[last] = '\n';
@@ -868,16 +877,16 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) {
     return OBJ_VAL(takeString(vm, temp, utf8size_lazy(temp)));
 }
 
-static Value isValidUtf8(DictuVM* vm, int argCount, Value* args) {
+static Value isValidUtf8(DictuVM *vm, int argCount, Value *args) {
     UNUSED(vm);
     UNUSED(argCount);
-    ObjString* string = AS_STRING(args[0]);
+    ObjString *string = AS_STRING(args[0]);
     return BOOL_VAL(string->character_len != -1);
 }
 
 void declareStringMethods(DictuVM *vm) {
-    // Note(Liz3): We need functions from the c stdlib for iswalpha, iswlower, iswupper(the utf8.c 
-    // library functions do not work)
+    // Note(Liz3): We need functions from the c stdlib for iswalpha, iswlower,
+    // iswupper(the utf8.c library functions do not work)
     setlocale(LC_ALL, "en_US.UTF-8");
     defineNative(vm, &vm->stringMethods, "len", lenString);
     defineNative(vm, &vm->stringMethods, "byteLen", byteLenString);
@@ -898,11 +907,13 @@ void declareStringMethods(DictuVM *vm) {
     defineNative(vm, &vm->stringMethods, "strip", stripString);
     defineNative(vm, &vm->stringMethods, "count", countString);
     defineNative(vm, &vm->stringMethods, "wordCount", wordCountString);
-    defineNative(vm, &vm->stringMethods, "toBool", boolNative); // Defined in util
+    defineNative(vm, &vm->stringMethods, "toBool",
+                 boolNative); // Defined in util
     defineNative(vm, &vm->stringMethods, "title", titleString);
     defineNative(vm, &vm->stringMethods, "repeat", repeatString);
     defineNative(vm, &vm->stringMethods, "isUpper", isUpperString);
     defineNative(vm, &vm->stringMethods, "isLower", isLowerString);
-    defineNative(vm, &vm->stringMethods, "collapseSpaces", collapseSpacesString);
+    defineNative(vm, &vm->stringMethods, "collapseSpaces",
+                 collapseSpacesString);
     defineNative(vm, &vm->stringMethods, "wrap", wrapString);
 }