diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt index 588a3ff33..5ed8e89b0 100644 --- a/src/cli/CMakeLists.txt +++ b/src/cli/CMakeLists.txt @@ -9,3 +9,7 @@ add_compile_definitions(USE_UTF8) add_executable(dictu ${DICTU_CLI_SRC}) target_include_directories(dictu PUBLIC ${INCLUDE_DIR}) target_link_libraries(dictu dictu_api_static) +if(LINUX AND CMAKE_C_COMPILER_ID STREQUAL "Clang") + # Clang needs this otherwise ld will fail on linux since the lib is build with -flto + set(CMAKE_C_FLAGS_RELEASE "-flto") +endif() \ No newline at end of file diff --git a/src/optionals/sqlite/sqlite3.c b/src/optionals/sqlite/sqlite3.c index 893718801..0955c8544 100644 --- a/src/optionals/sqlite/sqlite3.c +++ b/src/optionals/sqlite/sqlite3.c @@ -56382,7 +56382,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( const char *zUri = 0; /* URI args to copy */ int nUriByte = 1; /* Number of bytes of URI args at *zUri */ int nUri = 0; /* Number of URI parameters */ - + (void)nUri; /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). */ journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); diff --git a/src/optionals/uuid.c b/src/optionals/uuid.c index f0e3da6e7..28d5e324d 100644 --- a/src/optionals/uuid.c +++ b/src/optionals/uuid.c @@ -3,7 +3,9 @@ #ifndef _WIN32 static uuid_t uuid; -#define UUID_STRING_LEN 37 +#define UUID_STRING_LEN 36 // Note(Liz3): the strlen() of a UUID is 36, but the uuid_unparse_lower call expects + // a buffer with space for the null byte, but since copyString wants the strlen we + // keep the define to 36 and allocate +1. static Value uuidGenerateNative(DictuVM *vm, int argCount, Value *args) { UNUSED(args); @@ -21,7 +23,7 @@ static Value uuidGenerateNative(DictuVM *vm, int argCount, Value *args) { return newResultError(vm, "error: failed to generate uuid"); } - char out[UUID_STRING_LEN] = {}; + char out[UUID_STRING_LEN+1] = {}; uuid_unparse_lower(uuid, out); return newResultSuccess(vm, OBJ_VAL(copyString(vm, out, UUID_STRING_LEN))); @@ -43,7 +45,7 @@ static Value uuidGenRandomNative(DictuVM *vm, int argCount, Value *args) { return newResultError(vm, "error: failed to generate uuid"); } - char out[UUID_STRING_LEN] = {}; + char out[UUID_STRING_LEN+1] = {}; uuid_unparse_lower(uuid, out); return newResultSuccess(vm, OBJ_VAL(copyString(vm, out, UUID_STRING_LEN))); @@ -65,7 +67,7 @@ static Value uuidGenTimeNative(DictuVM *vm, int argCount, Value *args) { return newResultError(vm, "error: failed to generate uuid"); } - char out[UUID_STRING_LEN] = {}; + char out[UUID_STRING_LEN+1] = {}; uuid_unparse_lower(uuid, out); return newResultSuccess(vm, OBJ_VAL(copyString(vm, out, UUID_STRING_LEN))); diff --git a/src/vm/datatypes/strings.c b/src/vm/datatypes/strings.c index e58ca3e6d..d42440681 100644 --- a/src/vm/datatypes/strings.c +++ b/src/vm/datatypes/strings.c @@ -1,5 +1,11 @@ #include "strings.h" #include "../memory.h" +#include "../utf8.h" +#include +#include +#include +#include + static Value lenString(DictuVM *vm, int argCount, Value *args) { @@ -8,6 +14,21 @@ static Value lenString(DictuVM *vm, int argCount, Value *args) { return EMPTY_VAL; } + ObjString *string = AS_STRING(args[0]); + if (string->character_len == -1) { + runtimeError(vm, + "String contains invalid UTF-8, consider using byteLen()"); + return EMPTY_VAL; + } + return NUMBER_VAL(string->character_len); +} + +static Value byteLenString(DictuVM *vm, int argCount, Value *args) { + if (argCount != 0) { + runtimeError(vm, "byteLen() takes no arguments (%d given)", argCount); + return EMPTY_VAL; + } + ObjString *string = AS_STRING(args[0]); return NUMBER_VAL(string->length); } @@ -46,12 +67,19 @@ static Value toNumberString(DictuVM *vm, int argCount, Value *args) { static Value formatString(DictuVM *vm, int argCount, Value *args) { if (argCount == 0) { - runtimeError(vm, "format() takes at least 1 argument (%d given)", argCount); + runtimeError(vm, "format() takes at least 1 argument (%d given)", + argCount); + return EMPTY_VAL; + } + ObjString *string = AS_STRING(args[0]); + + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8", argCount); return EMPTY_VAL; } int length = 0; - char **replaceStrings = ALLOCATE(vm, char*, argCount); + char **replaceStrings = ALLOCATE(vm, char *, argCount); for (int j = 1; j < argCount + 1; j++) { Value value = args[j]; @@ -64,18 +92,14 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) { replaceStrings[j - 1] = str; } - length += strlen(replaceStrings[j - 1]); + length += utf8size_lazy(replaceStrings[j - 1]); } - ObjString *string = AS_STRING(args[0]); - - int stringLen = string->length + 1; - char *tmp = ALLOCATE(vm, char, stringLen); + char *tmp = string->chars; char *tmpFree = tmp; - memcpy(tmp, string->chars, stringLen); int count = 0; - while ((tmp = strstr(tmp, "{}"))) { + while ((tmp = utf8str(tmp, "{}"))) { count++; tmp++; } @@ -89,8 +113,7 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) { free(replaceStrings[i]); } - FREE_ARRAY(vm, char, tmp , stringLen); - FREE_ARRAY(vm, char*, replaceStrings, argCount); + FREE_ARRAY(vm, char *, replaceStrings, argCount); return EMPTY_VAL; } @@ -100,23 +123,21 @@ static Value formatString(DictuVM *vm, int argCount, Value *args) { int stringLength = 0; for (int i = 0; i < argCount; ++i) { - pos = strstr(tmp, "{}"); - if (pos != NULL) - *pos = '\0'; + pos = utf8str(tmp, "{}"); - int tmpLength = strlen(tmp); - int replaceLength = strlen(replaceStrings[i]); + int tmpLength = pos - tmp; + int replaceLength = utf8size_lazy(replaceStrings[i]); memcpy(newStr + stringLength, tmp, tmpLength); - memcpy(newStr + stringLength + tmpLength, replaceStrings[i], replaceLength); + memcpy(newStr + stringLength + tmpLength, replaceStrings[i], + replaceLength); stringLength += tmpLength + replaceLength; tmp = pos + 2; free(replaceStrings[i]); } - FREE_ARRAY(vm, char*, replaceStrings, argCount); - memcpy(newStr + stringLength, tmp, strlen(tmp)); + FREE_ARRAY(vm, char *, replaceStrings, argCount); + memcpy(newStr + stringLength, tmp, utf8size_lazy(tmp)); newStr[fullLength - 1] = '\0'; - FREE_ARRAY(vm, char, tmpFree, stringLen); return OBJ_VAL(takeString(vm, newStr, fullLength - 1)); } @@ -133,7 +154,17 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) { } ObjString *string = AS_STRING(args[0]); - char *delimiter = AS_CSTRING(args[1]); + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } + + ObjString *delimiterObj = AS_STRING(args[1]); + if (delimiterObj->character_len == -1) { + runtimeError(vm, "Delimiter String contains invalid UTF-8"); + return EMPTY_VAL; + } + char *delimiter = delimiterObj->chars; int maxSplit = string->length + 1; if (argCount == 2) { @@ -147,43 +178,45 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) { } } - char *tmp = ALLOCATE(vm, char, string->length + 1); - char *tmpFree = tmp; - memcpy(tmp, string->chars, string->length); - tmp[string->length] = '\0'; - int delimiterLength = strlen(delimiter); + char *tmp = string->chars; + int delimiterLength = utf8size_lazy(delimiter); char *token; ObjList *list = newList(vm); push(vm, OBJ_VAL(list)); int count = 0; - + char *ptr = string->chars; + utf8_int32_t cp; if (delimiterLength == 0) { int tokenIndex = 0; - for (; tokenIndex < string->length && count < maxSplit; tokenIndex++) { + for (; tokenIndex < string->character_len && count < maxSplit; + tokenIndex++) { count++; - *(tmp) = string->chars[tokenIndex]; - *(tmp + 1) = '\0'; - Value str = OBJ_VAL(copyString(vm, tmp, 1)); + + char *next_ptr = utf8codepoint(ptr, &cp); + + size_t size = next_ptr - ptr; + Value str = OBJ_VAL(copyString(vm, ptr, size)); // Push to stack to avoid GC push(vm, str); writeValueArray(vm, &list->values, str); pop(vm); + ptr = next_ptr; } - if (tokenIndex != string->length && count >= maxSplit) { - tmp = (string->chars) + tokenIndex; + if (tokenIndex != string->character_len && count >= maxSplit) { + tmp = ptr; } else { tmp = NULL; } } else if (maxSplit > 0) { do { count++; - token = strstr(tmp, delimiter); - if (token) - *token = '\0'; + token = utf8str(tmp, delimiter); - Value str = OBJ_VAL(copyString(vm, tmp, strlen(tmp))); + Value str = OBJ_VAL(copyString( + vm, tmp, + token == NULL ? utf8size_lazy(tmp) : ((size_t)(token - tmp)))); // Push to stack to avoid GC push(vm, str); @@ -199,7 +232,7 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) { } if (tmp != NULL && count >= maxSplit) { - Value remainingStr = OBJ_VAL(copyString(vm, tmp, strlen(tmp))); + Value remainingStr = OBJ_VAL(copyString(vm, tmp, utf8size_lazy(tmp))); // Push to stack to avoid GC push(vm, remainingStr); @@ -209,25 +242,36 @@ static Value splitString(DictuVM *vm, int argCount, Value *args) { pop(vm); - FREE_ARRAY(vm, char, tmpFree, string->length + 1); return OBJ_VAL(list); } static Value containsString(DictuVM *vm, int argCount, Value *args) { + if (argCount != 1) { runtimeError(vm, "contains() takes 1 argument (%d given)", argCount); return EMPTY_VAL; } + ObjString *strObj = AS_STRING(args[0]); + if (strObj->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } + if (!IS_STRING(args[1])) { runtimeError(vm, "Argument passed to contains() must be a string"); return EMPTY_VAL; } + ObjString *delimiterObj = AS_STRING(args[1]); + if (delimiterObj->character_len == -1) { + runtimeError(vm, "Search String contains invalid UTF-8"); + return EMPTY_VAL; + } - char *string = AS_CSTRING(args[0]); - char *delimiter = AS_CSTRING(args[1]); + char *string = strObj->chars; + char *delimiter = delimiterObj->chars; - if (!strstr(string, delimiter)) { + if (!utf8str(string, delimiter)) { return FALSE_VAL; } @@ -236,7 +280,14 @@ static Value containsString(DictuVM *vm, int argCount, Value *args) { static Value findString(DictuVM *vm, int argCount, Value *args) { if (argCount < 1 || argCount > 2) { - runtimeError(vm, "find() takes either 1 or 2 arguments (%d given)", argCount); + runtimeError(vm, "find() takes either 1 or 2 arguments (%d given)", + argCount); + return EMPTY_VAL; + } + + ObjString *strObj = AS_STRING(args[0]); + if (strObj->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); return EMPTY_VAL; } @@ -255,21 +306,28 @@ static Value findString(DictuVM *vm, int argCount, Value *args) { runtimeError(vm, "Substring passed to find() must be a string"); return EMPTY_VAL; } + ObjString *substrObj = AS_STRING(args[1]); + if (substrObj->character_len == -1) { + runtimeError(vm, "Substring String contains invalid UTF-8"); + return EMPTY_VAL; + } - char *substr = AS_CSTRING(args[1]); - char *string = AS_CSTRING(args[0]); + char *string = strObj->chars; + char *substr = substrObj->chars; int position = 0; for (int i = 0; i < index; ++i) { - char *result = strstr(string, substr); + if (i != 0) { + position += utf8len(substr); + } + char *result = utf8str(string, substr); if (!result) { position = -1; break; } - - position += (result - string) + (i * strlen(substr)); - string = result + strlen(substr); + position += utf8nlen(string, (result - string)); + string = result + utf8size_lazy(substr); } return NUMBER_VAL(position); @@ -280,42 +338,39 @@ static Value findLastString(DictuVM *vm, int argCount, Value *args) { runtimeError(vm, "findLast() takes 1 argument (%d given)", argCount); return EMPTY_VAL; } + if (!IS_STRING(args[1])) { + runtimeError(vm, "Substring passed to findLast() must be a string"); + return EMPTY_VAL; + } + utf8_int32_t cp; + ObjString *str = AS_STRING(args[0]); + ObjString *ss = AS_STRING(args[1]); + if (str->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } + if (ss->character_len == -1) { + runtimeError(vm, "Search String contains invalid UTF-8"); + return EMPTY_VAL; + } + if (ss->length > str->length || ss->character_len > str->character_len) + return NUMBER_VAL(-1); - const char *str = AS_CSTRING(args[0]); - const char *ss = AS_CSTRING(args[1]); - const char *p = str; - int found = !*ss; - - if (!found) { - while (*p) { - ++p; - } - - const char *q = ss; - while (*q) { - ++q; - } - - while (!found && !(p-str < q-ss)) { - const char *s = p; - const char *t = q; - - while (t != ss && *(s-1) == *(t-1)) { - --s; - --t; - } - - found = t == ss; + char *ptr = str->chars; + int index = str->character_len - ss->character_len; + for (int i = 0; i < index; i++) + ptr = utf8codepoint(ptr, &cp); - if (found) { - p = s; - } else { - --p; - } - } + while (true) { + if (utf8str(ptr, ss->chars) == ptr) + return NUMBER_VAL(index); + index--; + if (index < 0) + break; + ptr = utf8rcodepoint(ptr, &cp); } - return NUMBER_VAL(found ? p-str : -1); + return NUMBER_VAL(-1); } static Value replaceString(DictuVM *vm, int argCount, Value *args) { @@ -328,27 +383,36 @@ static Value replaceString(DictuVM *vm, int argCount, Value *args) { runtimeError(vm, "Arguments passed to replace() must be a strings"); return EMPTY_VAL; } - // Pop values off the stack - Value stringValue = args[0]; + ObjString *stringValue = AS_STRING(args[0]); + if (stringValue->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } + ObjString *to_replace = AS_STRING(args[1]); ObjString *replace = AS_STRING(args[2]); - char *string = AS_CSTRING(stringValue); + if (to_replace->character_len == -1) { + runtimeError(vm, "To Replace String contains invalid UTF-8"); + return EMPTY_VAL; + } + if (replace->character_len == -1) { + runtimeError(vm, "Replace String contains invalid UTF-8"); + return EMPTY_VAL; + } + char *string = stringValue->chars; int count = 0; int len = to_replace->length; int replaceLen = replace->length; - int stringLen = strlen(string) + 1; // Make a copy of the string so we do not modify the original - char *tmp = ALLOCATE(vm, char, stringLen + 1); + char *tmp = string; char *tmpFree = tmp; - memcpy(tmp, string, stringLen); - tmp[stringLen] = '\0'; // Count the occurrences of the needle so we can determine the size // of the string we need to allocate - while ((tmp = strstr(tmp, to_replace->chars)) != NULL) { + while ((tmp = utf8str(tmp, to_replace->chars)) != NULL) { count++; tmp += len; } @@ -357,29 +421,25 @@ static Value replaceString(DictuVM *vm, int argCount, Value *args) { tmp = tmpFree; if (count == 0) { - FREE_ARRAY(vm, char, tmpFree, stringLen + 1); - return stringValue; + return OBJ_VAL(stringValue); } - int length = strlen(tmp) - count * (len - replaceLen) + 1; + int length = utf8size_lazy(tmp) - count * (len - replaceLen) + 1; char *pos; char *newStr = ALLOCATE(vm, char, length); int stringLength = 0; for (int i = 0; i < count; ++i) { - pos = strstr(tmp, to_replace->chars); - if (pos != NULL) - *pos = '\0'; + pos = utf8str(tmp, to_replace->chars); - int tmpLength = strlen(tmp); + int tmpLength = pos - tmp; memcpy(newStr + stringLength, tmp, tmpLength); memcpy(newStr + stringLength + tmpLength, replace->chars, replaceLen); stringLength += tmpLength + replaceLen; tmp = pos + len; } - memcpy(newStr + stringLength, tmp, strlen(tmp)); - FREE_ARRAY(vm, char, tmpFree, stringLen + 1); + memcpy(newStr + stringLength, tmp, utf8size_lazy(tmp)); newStr[length - 1] = '\0'; return OBJ_VAL(takeString(vm, newStr, length - 1)); @@ -392,14 +452,17 @@ static Value lowerString(DictuVM *vm, int argCount, Value *args) { } ObjString *string = AS_STRING(args[0]); - char *temp = ALLOCATE(vm, char, string->length + 1); - - for (int i = 0; string->chars[i]; i++) { - temp[i] = tolower(string->chars[i]); + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; } + char *temp = ALLOCATE(vm, char, string->length + 1); + memcpy(temp, string->chars, string->length); temp[string->length] = '\0'; + utf8lwr(temp); - return OBJ_VAL(takeString(vm, temp, string->length)); + return OBJ_VAL( + takeStringWithLen(vm, temp, string->length, string->character_len)); } static Value upperString(DictuVM *vm, int argCount, Value *args) { @@ -409,14 +472,18 @@ static Value upperString(DictuVM *vm, int argCount, Value *args) { } ObjString *string = AS_STRING(args[0]); - char *temp = ALLOCATE(vm, char, string->length + 1); - - for (int i = 0; string->chars[i]; i++) { - temp[i] = toupper(string->chars[i]); + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; } + + char *temp = ALLOCATE(vm, char, string->length + 1); + memcpy(temp, string->chars, string->length); temp[string->length] = '\0'; + utf8upr(temp); - return OBJ_VAL(takeString(vm, temp, string->length)); + return OBJ_VAL( + takeStringWithLen(vm, temp, string->length, string->character_len)); } static Value startsWithString(DictuVM *vm, int argCount, Value *args) { @@ -430,10 +497,21 @@ static Value startsWithString(DictuVM *vm, int argCount, Value *args) { return EMPTY_VAL; } - char *string = AS_CSTRING(args[0]); + ObjString *string = AS_STRING(args[0]); + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } + ObjString *start = AS_STRING(args[1]); + if (start->character_len == -1) { + runtimeError(vm, "Prefix String contains invalid UTF-8"); + return EMPTY_VAL; + } + if (string->length < start->length) + return BOOL_VAL(false); - return BOOL_VAL(strncmp(string, start->chars, start->length) == 0); + return BOOL_VAL(utf8ncmp(string->chars, start->chars, start->length) == 0); } static Value endsWithString(DictuVM *vm, int argCount, Value *args) { @@ -448,13 +526,23 @@ static Value endsWithString(DictuVM *vm, int argCount, Value *args) { } ObjString *string = AS_STRING(args[0]); + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } + ObjString *suffix = AS_STRING(args[1]); + if (suffix->character_len == -1) { + runtimeError(vm, "Suffix String contains invalid UTF-8"); + return EMPTY_VAL; + } if (string->length < suffix->length) { return FALSE_VAL; } - return BOOL_VAL(strcmp(string->chars + (string->length - suffix->length), suffix->chars) == 0); + return BOOL_VAL(utf8cmp(string->chars + (string->length - suffix->length), + suffix->chars) == 0); } static Value leftStripString(DictuVM *vm, int argCount, Value *args) { @@ -475,7 +563,8 @@ static Value leftStripString(DictuVM *vm, int argCount, Value *args) { } if (count != 0) { - temp = SHRINK_ARRAY(vm, temp, char, string->length + 1, (string->length - count) + 1); + temp = SHRINK_ARRAY(vm, temp, char, string->length + 1, + (string->length - count) + 1); } memcpy(temp, string->chars + count, string->length - count); @@ -485,7 +574,8 @@ static Value leftStripString(DictuVM *vm, int argCount, Value *args) { static Value rightStripString(DictuVM *vm, int argCount, Value *args) { if (argCount != 0) { - runtimeError(vm, "rightStrip() takes no arguments (%d given)", argCount); + runtimeError(vm, "rightStrip() takes no arguments (%d given)", + argCount); return EMPTY_VAL; } @@ -533,11 +623,22 @@ static Value countString(DictuVM *vm, int argCount, Value *args) { return EMPTY_VAL; } - char *haystack = AS_CSTRING(args[0]); - char *needle = AS_CSTRING(args[1]); + ObjString *string = AS_STRING(args[0]); + ObjString *needleObj = AS_STRING(args[1]); + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } + if (needleObj->character_len == -1) { + runtimeError(vm, "Needle contains invalid UTF-8"); + return EMPTY_VAL; + } + + char *haystack = string->chars; + char *needle = needleObj->chars; int count = 0; - while ((haystack = strstr(haystack, needle))) { + while ((haystack = utf8str(haystack, needle))) { count++; haystack++; } @@ -551,17 +652,24 @@ static Value wordCountString(DictuVM *vm, int argCount, Value *args) { return EMPTY_VAL; } - char *string = AS_CSTRING(args[0]); - + ObjString *string = AS_STRING(args[0]); + + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } int count = 0; - int len = strlen(string); + int len = string->character_len; bool in = false; + char *ptr = string->chars; for (int i = 0; i < len; i++) { - if (isspace(string[i])) { + utf8_int32_t cp; + ptr = utf8codepoint(ptr, &cp); + if (iswspace(cp)) { in = false; - } else if(isalpha(string[i])) { - if(!in) { + } else if (iswalpha(cp)) { + if (!in) { in = true; count++; } @@ -578,25 +686,36 @@ static Value titleString(DictuVM *vm, int argCount, Value *args) { } ObjString *string = AS_STRING(args[0]); - char *temp = ALLOCATE(vm, char, string->length + 1); + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } - bool convertNext=true; + char *temp = ALLOCATE(vm, char, string->length + 1); + memcpy(temp, string->chars, string->length); + char *ptr = temp; + bool convertNext = true; + + for (int i = 0; i < string->character_len; i++) { + utf8_int32_t cp; + char *next_ptr = utf8codepoint(ptr, &cp); + if (cp == ' ') { + convertNext = true; + } else if (convertNext) { + cp = utf8uprcodepoint(cp); + convertNext = false; - for (int i = 0; string->chars[i]; i++) { - if(string->chars[i]==' '){ - convertNext=true; - } - else if(convertNext){ - temp[i] = toupper(string->chars[i]); - convertNext=false; - continue; + } else { + cp = utf8lwrcodepoint(cp); } - temp[i] = tolower(string->chars[i]); + utf8catcodepoint(ptr, cp, next_ptr - ptr); + ptr = next_ptr; } temp[string->length] = '\0'; - return OBJ_VAL(takeString(vm, temp, string->length)); + return OBJ_VAL( + takeStringWithLen(vm, temp, string->length, string->character_len)); } static Value repeatString(DictuVM *vm, int argCount, Value *args) { @@ -616,9 +735,9 @@ static Value repeatString(DictuVM *vm, int argCount, Value *args) { int tempLen = (string->length * count) + 1; char *temp = ALLOCATE(vm, char, tempLen); - strcpy(temp, string->chars); + utf8cpy(temp, string->chars); while (--count > 0) { - strcat(temp, string->chars); + utf8cat(temp, string->chars); } return OBJ_VAL(takeString(vm, temp, tempLen - 1)); @@ -630,19 +749,26 @@ static Value isUpperString(DictuVM *vm, int argCount, Value *args) { return EMPTY_VAL; } - char *string = AS_CSTRING(args[0]); - int len = strlen(string); + ObjString *strObj = AS_STRING(args[0]); + if (strObj->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } + char *string = strObj->chars; + int len = strObj->character_len; if (len == 0) { return BOOL_VAL(false); } for (int i = 0; i < len; i++) { - if (!isupper(string[i]) && isalpha(string[i])) { + utf8_int32_t cp; + string = utf8codepoint(string, &cp); + if (!iswupper(cp) && iswalpha(cp)) { return BOOL_VAL(false); } } - + return BOOL_VAL(true); } @@ -651,47 +777,67 @@ static Value isLowerString(DictuVM *vm, int argCount, Value *args) { runtimeError(vm, "isLower() takes no arguments (%d given)", argCount); return EMPTY_VAL; } + ObjString *strObj = AS_STRING(args[0]); + if (strObj->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } - char *string = AS_CSTRING(args[0]); - int len = strlen(string); + char *string = strObj->chars; + int len = strObj->character_len; if (len == 0) { return BOOL_VAL(false); } for (int i = 0; i < len; i++) { - if (!islower(string[i]) && isalpha(string[i])) { + utf8_int32_t cp; + string = utf8codepoint(string, &cp); + + if (!iswlower(cp) && iswalpha(cp)) { return BOOL_VAL(false); } } - + return BOOL_VAL(true); } static Value collapseSpacesString(DictuVM *vm, int argCount, Value *args) { if (argCount != 0) { - runtimeError(vm, "collapseSpaces() takes no arguments (%d given)", argCount); + runtimeError(vm, "collapseSpaces() takes no arguments (%d given)", + argCount); return EMPTY_VAL; } ObjString *string = AS_STRING(args[0]); - char *temp = ALLOCATE(vm, char, string->length + 1); - strcpy(temp, string->chars); - + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } + char *dest = ALLOCATE(vm, char, string->length + 1); + char *ptr = string->chars; + int targetLen = 0; int i, j; - for (i = j = 0; temp[i]; ++i) { - if (!isspace(temp[i]) || (i > 0 && !isspace(temp[i-1]))) { - temp[j++] = temp[i]; + utf8_int32_t last; + for (i = j = 0; i < string->character_len; ++i) { + utf8_int32_t cp; + char *n = utf8codepoint(ptr, &cp); + + if (!iswspace(cp) || (i > 0 && !iswspace(last))) { + utf8catcodepoint(dest + targetLen, cp, string->length - targetLen); + targetLen += utf8codepointsize(cp); } - } - temp[j+1] = '\0'; + last = cp; + ptr = n; + } - if (i != j) { - temp = SHRINK_ARRAY(vm, temp, char, string->length + 1, j + 1); + if (targetLen != string->length) { + dest = SHRINK_ARRAY(vm, dest, char, string->length + 1, targetLen + 1); } + dest[targetLen] = '\0'; - return OBJ_VAL(takeString(vm, temp, j)); + return OBJ_VAL(takeString(vm, dest, targetLen)); } static Value wrapString(DictuVM *vm, int argCount, Value *args) { @@ -701,31 +847,50 @@ static Value wrapString(DictuVM *vm, int argCount, Value *args) { } ObjString *string = AS_STRING(args[0]); + if (string->character_len == -1) { + runtimeError(vm, "String contains invalid UTF-8"); + return EMPTY_VAL; + } char *temp = ALLOCATE(vm, char, string->length + 1); - + temp[string->length] = '\0'; + memcpy(temp, string->chars, string->length); int len = AS_NUMBER(args[1]); - int last = 0; + int last = -1; int count = 0; + char *ptr = temp; + for (int cur = 0; cur < string->character_len; cur++, count++) { + utf8_int32_t cp; + char *n = utf8codepoint(ptr, &cp); - for (int cur = 0; string->chars[cur] != '\0'; cur++, count++) { - temp[cur] = string->chars[cur]; - - if (isspace(temp[cur])) { - last = cur; - } + if (isspace(cp)) { + last = ptr - temp; + } - if (count >= len) { + if (count >= len && last != -1) { temp[last] = '\n'; count = 0; } + ptr = n; } - return OBJ_VAL(takeString(vm, temp, strlen(temp))); + return OBJ_VAL(takeString(vm, temp, utf8size_lazy(temp))); +} + +static Value isValidUtf8(DictuVM *vm, int argCount, Value *args) { + UNUSED(vm); + UNUSED(argCount); + ObjString *string = AS_STRING(args[0]); + return BOOL_VAL(string->character_len != -1); } void declareStringMethods(DictuVM *vm) { + // Note(Liz3): We need functions from the c stdlib for iswalpha, iswlower, + // iswupper(the utf8.c library functions do not work) + setlocale(LC_ALL, "en_US.UTF-8"); defineNative(vm, &vm->stringMethods, "len", lenString); + defineNative(vm, &vm->stringMethods, "byteLen", byteLenString); + defineNative(vm, &vm->stringMethods, "isValidUtf8", isValidUtf8); defineNative(vm, &vm->stringMethods, "toNumber", toNumberString); defineNative(vm, &vm->stringMethods, "format", formatString); defineNative(vm, &vm->stringMethods, "split", splitString); @@ -742,11 +907,13 @@ void declareStringMethods(DictuVM *vm) { defineNative(vm, &vm->stringMethods, "strip", stripString); defineNative(vm, &vm->stringMethods, "count", countString); defineNative(vm, &vm->stringMethods, "wordCount", wordCountString); - defineNative(vm, &vm->stringMethods, "toBool", boolNative); // Defined in util + defineNative(vm, &vm->stringMethods, "toBool", + boolNative); // Defined in util defineNative(vm, &vm->stringMethods, "title", titleString); defineNative(vm, &vm->stringMethods, "repeat", repeatString); defineNative(vm, &vm->stringMethods, "isUpper", isUpperString); defineNative(vm, &vm->stringMethods, "isLower", isLowerString); - defineNative(vm, &vm->stringMethods, "collapseSpaces", collapseSpacesString); + defineNative(vm, &vm->stringMethods, "collapseSpaces", + collapseSpacesString); defineNative(vm, &vm->stringMethods, "wrap", wrapString); } diff --git a/src/vm/object.c b/src/vm/object.c index 413b29b8e..6324d02cd 100644 --- a/src/vm/object.c +++ b/src/vm/object.c @@ -7,6 +7,7 @@ #include "table.h" #include "value.h" #include "vm.h" +#include "utf8.h" #define ALLOCATE_OBJ(vm, type, objectType) \ (type*)allocateObject(vm, sizeof(type), objectType) @@ -146,18 +147,26 @@ ObjNative *newNative(DictuVM *vm, NativeFn function) { return native; } -static ObjString *allocateString(DictuVM *vm, char *chars, int length, - uint32_t hash) { +static ObjString *allocateStringWithLen(DictuVM *vm, char *chars, int length, + uint32_t hash, int character_len) { ObjString *string = ALLOCATE_OBJ(vm, ObjString, OBJ_STRING); string->length = length; string->chars = chars; string->hash = hash; + string->character_len = character_len; + push(vm, OBJ_VAL(string)); tableSet(vm, &vm->strings, string, NIL_VAL); pop(vm); return string; } +static ObjString *allocateString(DictuVM *vm, char *chars, int length, + uint32_t hash) { + int character_len = utf8valid(chars) == 0 ? (int)utf8len(chars) : -1; + return allocateStringWithLen(vm, chars, length, hash, character_len); +} + ObjList *newList(DictuVM *vm) { ObjList *list = ALLOCATE_OBJ(vm, ObjList, OBJ_LIST); initValueArray(&list->values); @@ -257,6 +266,20 @@ ObjString *takeString(DictuVM *vm, char *chars, int length) { return allocateString(vm, chars, length, hash); } +ObjString *takeStringWithLen(DictuVM *vm, char *chars, int length, int character_len) { + uint32_t hash = hashString(chars, length); + ObjString *interned = tableFindString(&vm->strings, chars, length, + hash); + if (interned != NULL) { + FREE_ARRAY(vm, char, chars, length + 1); + return interned; + } + + // Ensure terminating char is present + chars[length] = '\0'; + return allocateStringWithLen(vm, chars, length, hash, character_len); +} + ObjString *copyString(DictuVM *vm, const char *chars, int length) { uint32_t hash = hashString(chars, length); ObjString *interned = tableFindString(&vm->strings, chars, length, @@ -269,6 +292,18 @@ ObjString *copyString(DictuVM *vm, const char *chars, int length) { return allocateString(vm, heapChars, length, hash); } +ObjString *copyStringWithLen(DictuVM *vm, const char *chars, int length, int character_len) { + uint32_t hash = hashString(chars, length); + ObjString *interned = tableFindString(&vm->strings, chars, length, + hash); + if (interned != NULL) return interned; + + char *heapChars = ALLOCATE(vm, char, length + 1); + memcpy(heapChars, chars, length); + heapChars[length] = '\0'; + return allocateStringWithLen(vm, heapChars, length, hash, character_len); +} + ObjUpvalue *newUpvalue(DictuVM *vm, Value *slot) { ObjUpvalue *upvalue = ALLOCATE_OBJ(vm, ObjUpvalue, OBJ_UPVALUE); upvalue->closed = NIL_VAL; diff --git a/src/vm/object.h b/src/vm/object.h index 69ff5cc00..212876d7a 100644 --- a/src/vm/object.h +++ b/src/vm/object.h @@ -131,6 +131,7 @@ struct sObjString { int length; char *chars; uint32_t hash; + int character_len; }; struct sObjList { @@ -270,8 +271,10 @@ ObjInstance *newInstance(DictuVM *vm, ObjClass *klass); ObjNative *newNative(DictuVM *vm, NativeFn function); ObjString *takeString(DictuVM *vm, char *chars, int length); +ObjString *takeStringWithLen(DictuVM *vm, char *chars, int length, int character_len); ObjString *copyString(DictuVM *vm, const char *chars, int length); +ObjString *copyStringWithLen(DictuVM *vm, const char *chars, int length, int character_len); ObjList *newList(DictuVM *vm); diff --git a/src/vm/utf8.h b/src/vm/utf8.h new file mode 100644 index 000000000..495e259db --- /dev/null +++ b/src/vm/utf8.h @@ -0,0 +1,1712 @@ +/* The latest version of this library is available on GitHub; + * https://github.com/sheredom/utf8.h */ + +/* This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * For more information, please refer to */ + +#ifndef SHEREDOM_UTF8_H_INCLUDED +#define SHEREDOM_UTF8_H_INCLUDED + +#if defined(_MSC_VER) +#pragma warning(push) + +/* disable warning: no function prototype given: converting '()' to '(void)' */ +#pragma warning(disable : 4255) + +/* disable warning: '__cplusplus' is not defined as a preprocessor macro, + * replacing with '0' for '#if/#elif' */ +#pragma warning(disable : 4668) + +/* disable warning: bytes padding added after construct */ +#pragma warning(disable : 4820) +#endif + +#if defined(__cplusplus) +#if defined(_MSC_VER) +#define utf8_cplusplus _MSVC_LANG +#else +#define utf8_cplusplus __cplusplus +#endif +#endif + +#include +#include + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(_MSC_VER) && (_MSC_VER < 1920) +typedef __int32 utf8_int32_t; +#else +#include +typedef int32_t utf8_int32_t; +#endif + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wold-style-cast" +#pragma clang diagnostic ignored "-Wcast-qual" + +#if __has_warning("-Wunsafe-buffer-usage") +#pragma clang diagnostic ignored "-Wunsafe-buffer-usage" +#endif +#endif + +#ifdef utf8_cplusplus +extern "C" { +#endif + +#if defined(__TINYC__) +#define UTF8_ATTRIBUTE(a) __attribute((a)) +#else +#define UTF8_ATTRIBUTE(a) __attribute__((a)) +#endif + +#if defined(_MSC_VER) +#define utf8_nonnull +#define utf8_pure +#define utf8_restrict __restrict +#define utf8_weak __inline +#elif defined(__clang__) || defined(__GNUC__) +#define utf8_nonnull UTF8_ATTRIBUTE(nonnull) +#define utf8_pure UTF8_ATTRIBUTE(pure) +#define utf8_restrict __restrict__ +#define utf8_weak UTF8_ATTRIBUTE(weak) +#elif defined(__TINYC__) +#define utf8_nonnull UTF8_ATTRIBUTE(nonnull) +#define utf8_pure UTF8_ATTRIBUTE(pure) +#define utf8_restrict +#define utf8_weak UTF8_ATTRIBUTE(weak) +#else +#error Non clang, non gcc, non MSVC, non tcc compiler found! +#endif + +#ifdef utf8_cplusplus +#define utf8_null NULL +#else +#define utf8_null 0 +#endif + +#if defined(utf8_cplusplus) && utf8_cplusplus >= 201402L && (!defined(_MSC_VER) || (defined(_MSC_VER) && _MSC_VER >= 1910)) +#define utf8_constexpr14 constexpr +#define utf8_constexpr14_impl constexpr +#else +/* constexpr and weak are incompatible. so only enable one of them */ +#define utf8_constexpr14 utf8_weak +#define utf8_constexpr14_impl +#endif + +#if defined(utf8_cplusplus) && utf8_cplusplus >= 202002L +using utf8_int8_t = char8_t; /* Introduced in C++20 */ +#else +typedef char utf8_int8_t; +#endif + +/* Return less than 0, 0, greater than 0 if src1 < src2, src1 == src2, src1 > + * src2 respectively, case insensitive. */ +utf8_constexpr14 utf8_nonnull utf8_pure int +utf8casecmp(const utf8_int8_t *src1, const utf8_int8_t *src2); + +/* Append the utf8 string src onto the utf8 string dst. */ +utf8_nonnull utf8_weak utf8_int8_t * +utf8cat(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src); + +/* Find the first match of the utf8 codepoint chr in the utf8 string src. */ +utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t * +utf8chr(const utf8_int8_t *src, utf8_int32_t chr); + +/* Return less than 0, 0, greater than 0 if src1 < src2, + * src1 == src2, src1 > src2 respectively. */ +utf8_constexpr14 utf8_nonnull utf8_pure int utf8cmp(const utf8_int8_t *src1, + const utf8_int8_t *src2); + +/* Copy the utf8 string src onto the memory allocated in dst. */ +utf8_nonnull utf8_weak utf8_int8_t * +utf8cpy(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src); + +/* Number of utf8 codepoints in the utf8 string src that consists entirely + * of utf8 codepoints not from the utf8 string reject. */ +utf8_constexpr14 utf8_nonnull utf8_pure size_t +utf8cspn(const utf8_int8_t *src, const utf8_int8_t *reject); + +/* Duplicate the utf8 string src by getting its size, malloc'ing a new buffer + * copying over the data, and returning that. Or 0 if malloc failed. */ +utf8_weak utf8_int8_t *utf8dup(const utf8_int8_t *src); + +/* Number of utf8 codepoints in the utf8 string str, + * excluding the null terminating byte. */ +utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8len(const utf8_int8_t *str); + +/* Similar to utf8len, except that only at most n bytes of src are looked. */ +utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8nlen(const utf8_int8_t *str, + size_t n); + +/* Return less than 0, 0, greater than 0 if src1 < src2, src1 == src2, src1 > + * src2 respectively, case insensitive. Checking at most n bytes of each utf8 + * string. */ +utf8_constexpr14 utf8_nonnull utf8_pure int +utf8ncasecmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n); + +/* Append the utf8 string src onto the utf8 string dst, + * writing at most n+1 bytes. Can produce an invalid utf8 + * string if n falls partway through a utf8 codepoint. */ +utf8_nonnull utf8_weak utf8_int8_t * +utf8ncat(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src, + size_t n); + +/* Return less than 0, 0, greater than 0 if src1 < src2, + * src1 == src2, src1 > src2 respectively. Checking at most n + * bytes of each utf8 string. */ +utf8_constexpr14 utf8_nonnull utf8_pure int +utf8ncmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n); + +/* Copy the utf8 string src onto the memory allocated in dst. + * Copies at most n bytes. If n falls partway through a utf8 + * codepoint, or if dst doesn't have enough room for a null + * terminator, the final string will be cut short to preserve + * utf8 validity. */ + +utf8_nonnull utf8_weak utf8_int8_t * +utf8ncpy(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src, + size_t n); + +/* Similar to utf8dup, except that at most n bytes of src are copied. If src is + * longer than n, only n bytes are copied and a null byte is added. + * + * Returns a new string if successful, 0 otherwise */ +utf8_weak utf8_int8_t *utf8ndup(const utf8_int8_t *src, size_t n); + +/* Locates the first occurrence in the utf8 string str of any byte in the + * utf8 string accept, or 0 if no match was found. */ +utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t * +utf8pbrk(const utf8_int8_t *str, const utf8_int8_t *accept); + +/* Find the last match of the utf8 codepoint chr in the utf8 string src. */ +utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t * +utf8rchr(const utf8_int8_t *src, int chr); + +/* Number of bytes in the utf8 string str, + * including the null terminating byte. */ +utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8size(const utf8_int8_t *str); + +/* Similar to utf8size, except that the null terminating byte is excluded. */ +utf8_constexpr14 utf8_nonnull utf8_pure size_t +utf8size_lazy(const utf8_int8_t *str); + +/* Similar to utf8size, except that only at most n bytes of src are looked and + * the null terminating byte is excluded. */ +utf8_constexpr14 utf8_nonnull utf8_pure size_t +utf8nsize_lazy(const utf8_int8_t *str, size_t n); + +/* Number of utf8 codepoints in the utf8 string src that consists entirely + * of utf8 codepoints from the utf8 string accept. */ +utf8_constexpr14 utf8_nonnull utf8_pure size_t +utf8spn(const utf8_int8_t *src, const utf8_int8_t *accept); + +/* The position of the utf8 string needle in the utf8 string haystack. */ +utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t * +utf8str(const utf8_int8_t *haystack, const utf8_int8_t *needle); + +/* The position of the utf8 string needle in the utf8 string haystack, case + * insensitive. */ +utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t * +utf8casestr(const utf8_int8_t *haystack, const utf8_int8_t *needle); + +/* Return 0 on success, or the position of the invalid + * utf8 codepoint on failure. */ +utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t * +utf8valid(const utf8_int8_t *str); + +/* Similar to utf8valid, except that only at most n bytes of src are looked. */ +utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t * +utf8nvalid(const utf8_int8_t *str, size_t n); + +/* Given a null-terminated string, makes the string valid by replacing invalid + * codepoints with a 1-byte replacement. Returns 0 on success. */ +utf8_nonnull utf8_weak int utf8makevalid(utf8_int8_t *str, + const utf8_int32_t replacement); + +/* Sets out_codepoint to the current utf8 codepoint in str, and returns the + * address of the next utf8 codepoint after the current one in str. */ +utf8_constexpr14 utf8_nonnull utf8_int8_t * +utf8codepoint(const utf8_int8_t *utf8_restrict str, + utf8_int32_t *utf8_restrict out_codepoint); + +/* Calculates the size of the next utf8 codepoint in str. */ +utf8_constexpr14 utf8_nonnull size_t +utf8codepointcalcsize(const utf8_int8_t *str); + +/* Returns the size of the given codepoint in bytes. */ +utf8_constexpr14 size_t utf8codepointsize(utf8_int32_t chr); + +/* Write a codepoint to the given string, and return the address to the next + * place after the written codepoint. Pass how many bytes left in the buffer to + * n. If there is not enough space for the codepoint, this function returns + * null. */ +utf8_nonnull utf8_weak utf8_int8_t * +utf8catcodepoint(utf8_int8_t *str, utf8_int32_t chr, size_t n); + +/* Returns 1 if the given character is lowercase, or 0 if it is not. */ +utf8_constexpr14 int utf8islower(utf8_int32_t chr); + +/* Returns 1 if the given character is uppercase, or 0 if it is not. */ +utf8_constexpr14 int utf8isupper(utf8_int32_t chr); + +/* Transform the given string into all lowercase codepoints. */ +utf8_nonnull utf8_weak void utf8lwr(utf8_int8_t *utf8_restrict str); + +/* Transform the given string into all uppercase codepoints. */ +utf8_nonnull utf8_weak void utf8upr(utf8_int8_t *utf8_restrict str); + +/* Make a codepoint lower case if possible. */ +utf8_constexpr14 utf8_int32_t utf8lwrcodepoint(utf8_int32_t cp); + +/* Make a codepoint upper case if possible. */ +utf8_constexpr14 utf8_int32_t utf8uprcodepoint(utf8_int32_t cp); + +/* Sets out_codepoint to the current utf8 codepoint in str, and returns the + * address of the previous utf8 codepoint before the current one in str. */ +utf8_constexpr14 utf8_nonnull utf8_int8_t * +utf8rcodepoint(const utf8_int8_t *utf8_restrict str, + utf8_int32_t *utf8_restrict out_codepoint); + +/* Duplicate the utf8 string src by getting its size, calling alloc_func_ptr to + * copy over data to a new buffer, and returning that. Or 0 if alloc_func_ptr + * returned null. */ +utf8_weak utf8_int8_t *utf8dup_ex(const utf8_int8_t *src, + utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, + size_t), + utf8_int8_t *user_data); + +/* Similar to utf8dup, except that at most n bytes of src are copied. If src is + * longer than n, only n bytes are copied and a null byte is added. + * + * Returns a new string if successful, 0 otherwise. */ +utf8_weak utf8_int8_t *utf8ndup_ex(const utf8_int8_t *src, size_t n, + utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, + size_t), + utf8_int8_t *user_data); + +#undef utf8_weak +#undef utf8_pure +#undef utf8_nonnull + +utf8_constexpr14_impl int utf8casecmp(const utf8_int8_t *src1, + const utf8_int8_t *src2) { + utf8_int32_t src1_lwr_cp = 0, src2_lwr_cp = 0, src1_upr_cp = 0, + src2_upr_cp = 0, src1_orig_cp = 0, src2_orig_cp = 0; + + for (;;) { + src1 = utf8codepoint(src1, &src1_orig_cp); + src2 = utf8codepoint(src2, &src2_orig_cp); + + /* lower the srcs if required */ + src1_lwr_cp = utf8lwrcodepoint(src1_orig_cp); + src2_lwr_cp = utf8lwrcodepoint(src2_orig_cp); + + /* lower the srcs if required */ + src1_upr_cp = utf8uprcodepoint(src1_orig_cp); + src2_upr_cp = utf8uprcodepoint(src2_orig_cp); + + /* check if the lowered codepoints match */ + if ((0 == src1_orig_cp) && (0 == src2_orig_cp)) { + return 0; + } else if ((src1_lwr_cp == src2_lwr_cp) || (src1_upr_cp == src2_upr_cp)) { + continue; + } + + /* if they don't match, then we return the difference between the characters + */ + return src1_lwr_cp - src2_lwr_cp; + } +} + +utf8_int8_t *utf8cat(utf8_int8_t *utf8_restrict dst, + const utf8_int8_t *utf8_restrict src) { + utf8_int8_t *d = dst; + /* find the null terminating byte in dst */ + while ('\0' != *d) { + d++; + } + + /* overwriting the null terminating byte in dst, append src byte-by-byte */ + while ('\0' != *src) { + *d++ = *src++; + } + + /* write out a new null terminating byte into dst */ + *d = '\0'; + + return dst; +} + +utf8_constexpr14_impl utf8_int8_t *utf8chr(const utf8_int8_t *src, + utf8_int32_t chr) { + utf8_int8_t c[5] = {'\0', '\0', '\0', '\0', '\0'}; + + if (0 == chr) { + /* being asked to return position of null terminating byte, so + * just run s to the end, and return! */ + while ('\0' != *src) { + src++; + } + return (utf8_int8_t *)src; + } else if (0 == ((utf8_int32_t)0xffffff80 & chr)) { + /* 1-byte/7-bit ascii + * (0b0xxxxxxx) */ + c[0] = (utf8_int8_t)chr; + } else if (0 == ((utf8_int32_t)0xfffff800 & chr)) { + /* 2-byte/11-bit utf8 code point + * (0b110xxxxx 0b10xxxxxx) */ + c[0] = (utf8_int8_t)(0xc0 | (utf8_int8_t)(chr >> 6)); + c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f)); + } else if (0 == ((utf8_int32_t)0xffff0000 & chr)) { + /* 3-byte/16-bit utf8 code point + * (0b1110xxxx 0b10xxxxxx 0b10xxxxxx) */ + c[0] = (utf8_int8_t)(0xe0 | (utf8_int8_t)(chr >> 12)); + c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f)); + c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f)); + } else { /* if (0 == ((int)0xffe00000 & chr)) { */ + /* 4-byte/21-bit utf8 code point + * (0b11110xxx 0b10xxxxxx 0b10xxxxxx 0b10xxxxxx) */ + c[0] = (utf8_int8_t)(0xf0 | (utf8_int8_t)(chr >> 18)); + c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 12) & 0x3f)); + c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f)); + c[3] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f)); + } + + /* we've made c into a 2 utf8 codepoint string, one for the chr we are + * seeking, another for the null terminating byte. Now use utf8str to + * search */ + return utf8str(src, c); +} + +utf8_constexpr14_impl int utf8cmp(const utf8_int8_t *src1, + const utf8_int8_t *src2) { + while (('\0' != *src1) || ('\0' != *src2)) { + if (*src1 < *src2) { + return -1; + } else if (*src1 > *src2) { + return 1; + } + + src1++; + src2++; + } + + /* both utf8 strings matched */ + return 0; +} + +utf8_constexpr14_impl int utf8coll(const utf8_int8_t *src1, + const utf8_int8_t *src2); + +utf8_int8_t *utf8cpy(utf8_int8_t *utf8_restrict dst, + const utf8_int8_t *utf8_restrict src) { + utf8_int8_t *d = dst; + + /* overwriting anything previously in dst, write byte-by-byte + * from src */ + while ('\0' != *src) { + *d++ = *src++; + } + + /* append null terminating byte */ + *d = '\0'; + + return dst; +} + +utf8_constexpr14_impl size_t utf8cspn(const utf8_int8_t *src, + const utf8_int8_t *reject) { + size_t chars = 0; + + while ('\0' != *src) { + const utf8_int8_t *r = reject; + size_t offset = 0; + + while ('\0' != *r) { + /* checking that if *r is the start of a utf8 codepoint + * (it is not 0b10xxxxxx) and we have successfully matched + * a previous character (0 < offset) - we found a match */ + if ((0x80 != (0xc0 & *r)) && (0 < offset)) { + return chars; + } else { + if (*r == src[offset]) { + /* part of a utf8 codepoint matched, so move our checking + * onwards to the next byte */ + offset++; + r++; + } else { + /* r could be in the middle of an unmatching utf8 code point, + * so we need to march it on to the next character beginning, */ + + do { + r++; + } while (0x80 == (0xc0 & *r)); + + /* reset offset too as we found a mismatch */ + offset = 0; + } + } + } + + /* found a match at the end of *r, so didn't get a chance to test it */ + if (0 < offset) { + return chars; + } + + /* the current utf8 codepoint in src did not match reject, but src + * could have been partway through a utf8 codepoint, so we need to + * march it onto the next utf8 codepoint starting byte */ + do { + src++; + } while ((0x80 == (0xc0 & *src))); + chars++; + } + + return chars; +} + +utf8_int8_t *utf8dup(const utf8_int8_t *src) { + return utf8dup_ex(src, utf8_null, utf8_null); +} + +utf8_int8_t *utf8dup_ex(const utf8_int8_t *src, + utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, size_t), + utf8_int8_t *user_data) { + utf8_int8_t *n = utf8_null; + + /* figure out how many bytes (including the terminator) we need to copy first + */ + size_t bytes = utf8size(src); + + if (alloc_func_ptr) { + n = alloc_func_ptr(user_data, bytes); + } else { +#if !defined(UTF8_NO_STD_MALLOC) + n = (utf8_int8_t *)malloc(bytes); +#else + return utf8_null; +#endif + } + + if (utf8_null == n) { + /* out of memory so we bail */ + return utf8_null; + } else { + bytes = 0; + + /* copy src byte-by-byte into our new utf8 string */ + while ('\0' != src[bytes]) { + n[bytes] = src[bytes]; + bytes++; + } + + /* append null terminating byte */ + n[bytes] = '\0'; + return n; + } +} + +utf8_constexpr14_impl utf8_int8_t *utf8fry(const utf8_int8_t *str); + +utf8_constexpr14_impl size_t utf8len(const utf8_int8_t *str) { + return utf8nlen(str, SIZE_MAX); +} + +utf8_constexpr14_impl size_t utf8nlen(const utf8_int8_t *str, size_t n) { + const utf8_int8_t *t = str; + size_t length = 0; + + while ((size_t)(str - t) < n && '\0' != *str) { + if (0xf0 == (0xf8 & *str)) { + /* 4-byte utf8 code point (began with 0b11110xxx) */ + str += 4; + } else if (0xe0 == (0xf0 & *str)) { + /* 3-byte utf8 code point (began with 0b1110xxxx) */ + str += 3; + } else if (0xc0 == (0xe0 & *str)) { + /* 2-byte utf8 code point (began with 0b110xxxxx) */ + str += 2; + } else { /* if (0x00 == (0x80 & *s)) { */ + /* 1-byte ascii (began with 0b0xxxxxxx) */ + str += 1; + } + + /* no matter the bytes we marched s forward by, it was + * only 1 utf8 codepoint */ + length++; + } + + if ((size_t)(str - t) > n) { + length--; + } + return length; +} + +utf8_constexpr14_impl int utf8ncasecmp(const utf8_int8_t *src1, + const utf8_int8_t *src2, size_t n) { + utf8_int32_t src1_lwr_cp = 0, src2_lwr_cp = 0, src1_upr_cp = 0, + src2_upr_cp = 0, src1_orig_cp = 0, src2_orig_cp = 0; + + do { + const utf8_int8_t *const s1 = src1; + const utf8_int8_t *const s2 = src2; + + /* first check that we have enough bytes left in n to contain an entire + * codepoint */ + if (0 == n) { + return 0; + } + + if ((1 == n) && ((0xc0 == (0xe0 & *s1)) || (0xc0 == (0xe0 & *s2)))) { + const utf8_int32_t c1 = (0xe0 & *s1); + const utf8_int32_t c2 = (0xe0 & *s2); + + if (c1 != c2) { + return c1 - c2; + } else { + return 0; + } + } + + if ((2 >= n) && ((0xe0 == (0xf0 & *s1)) || (0xe0 == (0xf0 & *s2)))) { + const utf8_int32_t c1 = (0xf0 & *s1); + const utf8_int32_t c2 = (0xf0 & *s2); + + if (c1 != c2) { + return c1 - c2; + } else { + return 0; + } + } + + if ((3 >= n) && ((0xf0 == (0xf8 & *s1)) || (0xf0 == (0xf8 & *s2)))) { + const utf8_int32_t c1 = (0xf8 & *s1); + const utf8_int32_t c2 = (0xf8 & *s2); + + if (c1 != c2) { + return c1 - c2; + } else { + return 0; + } + } + + src1 = utf8codepoint(src1, &src1_orig_cp); + src2 = utf8codepoint(src2, &src2_orig_cp); + n -= utf8codepointsize(src1_orig_cp); + + src1_lwr_cp = utf8lwrcodepoint(src1_orig_cp); + src2_lwr_cp = utf8lwrcodepoint(src2_orig_cp); + + src1_upr_cp = utf8uprcodepoint(src1_orig_cp); + src2_upr_cp = utf8uprcodepoint(src2_orig_cp); + + /* check if the lowered codepoints match */ + if ((0 == src1_orig_cp) && (0 == src2_orig_cp)) { + return 0; + } else if ((src1_lwr_cp == src2_lwr_cp) || (src1_upr_cp == src2_upr_cp)) { + continue; + } + + /* if they don't match, then we return the difference between the characters + */ + return src1_lwr_cp - src2_lwr_cp; + } while (0 < n); + + /* both utf8 strings matched */ + return 0; +} + +utf8_int8_t *utf8ncat(utf8_int8_t *utf8_restrict dst, + const utf8_int8_t *utf8_restrict src, size_t n) { + utf8_int8_t *d = dst; + + /* find the null terminating byte in dst */ + while ('\0' != *d) { + d++; + } + + /* overwriting the null terminating byte in dst, append src byte-by-byte + * stopping if we run out of space */ + while (('\0' != *src) && (0 != n--)) { + *d++ = *src++; + } + + /* write out a new null terminating byte into dst */ + *d = '\0'; + + return dst; +} + +utf8_constexpr14_impl int utf8ncmp(const utf8_int8_t *src1, + const utf8_int8_t *src2, size_t n) { + while ((0 != n--) && (('\0' != *src1) || ('\0' != *src2))) { + if (*src1 < *src2) { + return -1; + } else if (*src1 > *src2) { + return 1; + } + + src1++; + src2++; + } + + /* both utf8 strings matched */ + return 0; +} + +utf8_int8_t *utf8ncpy(utf8_int8_t *utf8_restrict dst, + const utf8_int8_t *utf8_restrict src, size_t n) { + utf8_int8_t *d = dst; + size_t index = 0, check_index = 0; + + if (n == 0) { + return dst; + } + + /* overwriting anything previously in dst, write byte-by-byte + * from src */ + for (index = 0; index < n; index++) { + d[index] = src[index]; + if ('\0' == src[index]) { + break; + } + } + + for (check_index = index - 1; + check_index > 0 && 0x80 == (0xc0 & d[check_index]); check_index--) { + /* just moving the index */ + } + + if (check_index < index && + ((index - check_index) < utf8codepointcalcsize(&d[check_index]) || + (index - check_index) == n)) { + index = check_index; + } + + /* append null terminating byte */ + for (; index < n; index++) { + d[index] = 0; + } + + return dst; +} + +utf8_int8_t *utf8ndup(const utf8_int8_t *src, size_t n) { + return utf8ndup_ex(src, n, utf8_null, utf8_null); +} + +utf8_int8_t *utf8ndup_ex(const utf8_int8_t *src, size_t n, + utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, size_t), + utf8_int8_t *user_data) { + utf8_int8_t *c = utf8_null; + size_t bytes = 0; + + /* Find the end of the string or stop when n is reached */ + while ('\0' != src[bytes] && bytes < n) { + bytes++; + } + + /* In case bytes is actually less than n, we need to set it + * to be used later in the copy byte by byte. */ + n = bytes; + + if (alloc_func_ptr) { + c = alloc_func_ptr(user_data, bytes + 1); + } else { +#if !defined(UTF8_NO_STD_MALLOC) + c = (utf8_int8_t *)malloc(bytes + 1); +#else + c = utf8_null; +#endif + } + + if (utf8_null == c) { + /* out of memory so we bail */ + return utf8_null; + } + + bytes = 0; + + /* copy src byte-by-byte into our new utf8 string */ + while ('\0' != src[bytes] && bytes < n) { + c[bytes] = src[bytes]; + bytes++; + } + + /* append null terminating byte */ + c[bytes] = '\0'; + return c; +} + +utf8_constexpr14_impl utf8_int8_t *utf8rchr(const utf8_int8_t *src, int chr) { + + utf8_int8_t *match = utf8_null; + utf8_int8_t c[5] = {'\0', '\0', '\0', '\0', '\0'}; + + if (0 == chr) { + /* being asked to return position of null terminating byte, so + * just run s to the end, and return! */ + while ('\0' != *src) { + src++; + } + return (utf8_int8_t *)src; + } else if (0 == ((int)0xffffff80 & chr)) { + /* 1-byte/7-bit ascii + * (0b0xxxxxxx) */ + c[0] = (utf8_int8_t)chr; + } else if (0 == ((int)0xfffff800 & chr)) { + /* 2-byte/11-bit utf8 code point + * (0b110xxxxx 0b10xxxxxx) */ + c[0] = (utf8_int8_t)(0xc0 | (utf8_int8_t)(chr >> 6)); + c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f)); + } else if (0 == ((int)0xffff0000 & chr)) { + /* 3-byte/16-bit utf8 code point + * (0b1110xxxx 0b10xxxxxx 0b10xxxxxx) */ + c[0] = (utf8_int8_t)(0xe0 | (utf8_int8_t)(chr >> 12)); + c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f)); + c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f)); + } else { /* if (0 == ((int)0xffe00000 & chr)) { */ + /* 4-byte/21-bit utf8 code point + * (0b11110xxx 0b10xxxxxx 0b10xxxxxx 0b10xxxxxx) */ + c[0] = (utf8_int8_t)(0xf0 | (utf8_int8_t)(chr >> 18)); + c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 12) & 0x3f)); + c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f)); + c[3] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f)); + } + + /* we've created a 2 utf8 codepoint string in c that is + * the utf8 character asked for by chr, and a null + * terminating byte */ + + while ('\0' != *src) { + size_t offset = 0; + + while ((src[offset] == c[offset]) && ('\0' != src[offset])) { + offset++; + } + + if ('\0' == c[offset]) { + /* we found a matching utf8 code point */ + match = (utf8_int8_t *)src; + src += offset; + + if ('\0' == *src) { + break; + } + } else { + src += offset; + + /* need to march s along to next utf8 codepoint start + * (the next byte that doesn't match 0b10xxxxxx) */ + if ('\0' != *src) { + do { + src++; + } while (0x80 == (0xc0 & *src)); + } + } + } + + /* return the last match we found (or 0 if no match was found) */ + return match; +} + +utf8_constexpr14_impl utf8_int8_t *utf8pbrk(const utf8_int8_t *str, + const utf8_int8_t *accept) { + while ('\0' != *str) { + const utf8_int8_t *a = accept; + size_t offset = 0; + + while ('\0' != *a) { + /* checking that if *a is the start of a utf8 codepoint + * (it is not 0b10xxxxxx) and we have successfully matched + * a previous character (0 < offset) - we found a match */ + if ((0x80 != (0xc0 & *a)) && (0 < offset)) { + return (utf8_int8_t *)str; + } else { + if (*a == str[offset]) { + /* part of a utf8 codepoint matched, so move our checking + * onwards to the next byte */ + offset++; + a++; + } else { + /* r could be in the middle of an unmatching utf8 code point, + * so we need to march it on to the next character beginning, */ + + do { + a++; + } while (0x80 == (0xc0 & *a)); + + /* reset offset too as we found a mismatch */ + offset = 0; + } + } + } + + /* we found a match on the last utf8 codepoint */ + if (0 < offset) { + return (utf8_int8_t *)str; + } + + /* the current utf8 codepoint in src did not match accept, but src + * could have been partway through a utf8 codepoint, so we need to + * march it onto the next utf8 codepoint starting byte */ + do { + str++; + } while ((0x80 == (0xc0 & *str))); + } + + return utf8_null; +} + +utf8_constexpr14_impl size_t utf8size(const utf8_int8_t *str) { + return utf8size_lazy(str) + 1; +} + +utf8_constexpr14_impl size_t utf8size_lazy(const utf8_int8_t *str) { + return utf8nsize_lazy(str, SIZE_MAX); +} + +utf8_constexpr14_impl size_t utf8nsize_lazy(const utf8_int8_t *str, size_t n) { + size_t size = 0; + while (size < n && '\0' != str[size]) { + size++; + } + return size; +} + +utf8_constexpr14_impl size_t utf8spn(const utf8_int8_t *src, + const utf8_int8_t *accept) { + size_t chars = 0; + + while ('\0' != *src) { + const utf8_int8_t *a = accept; + size_t offset = 0; + + while ('\0' != *a) { + /* checking that if *r is the start of a utf8 codepoint + * (it is not 0b10xxxxxx) and we have successfully matched + * a previous character (0 < offset) - we found a match */ + if ((0x80 != (0xc0 & *a)) && (0 < offset)) { + /* found a match, so increment the number of utf8 codepoints + * that have matched and stop checking whether any other utf8 + * codepoints in a match */ + chars++; + src += offset; + offset = 0; + break; + } else { + if (*a == src[offset]) { + offset++; + a++; + } else { + /* a could be in the middle of an unmatching utf8 codepoint, + * so we need to march it on to the next character beginning, */ + do { + a++; + } while (0x80 == (0xc0 & *a)); + + /* reset offset too as we found a mismatch */ + offset = 0; + } + } + } + + /* found a match at the end of *a, so didn't get a chance to test it */ + if (0 < offset) { + chars++; + src += offset; + continue; + } + + /* if a got to its terminating null byte, then we didn't find a match. + * Return the current number of matched utf8 codepoints */ + if ('\0' == *a) { + return chars; + } + } + + return chars; +} + +utf8_constexpr14_impl utf8_int8_t *utf8str(const utf8_int8_t *haystack, + const utf8_int8_t *needle) { + utf8_int32_t throwaway_codepoint = 0; + + /* if needle has no utf8 codepoints before the null terminating + * byte then return haystack */ + if ('\0' == *needle) { + return (utf8_int8_t *)haystack; + } + + while ('\0' != *haystack) { + const utf8_int8_t *maybeMatch = haystack; + const utf8_int8_t *n = needle; + + while (*haystack == *n && (*haystack != '\0' && *n != '\0')) { + n++; + haystack++; + } + + if ('\0' == *n) { + /* we found the whole utf8 string for needle in haystack at + * maybeMatch, so return it */ + return (utf8_int8_t *)maybeMatch; + } else { + /* h could be in the middle of an unmatching utf8 codepoint, + * so we need to march it on to the next character beginning + * starting from the current character */ + haystack = utf8codepoint(maybeMatch, &throwaway_codepoint); + } + } + + /* no match */ + return utf8_null; +} + +utf8_constexpr14_impl utf8_int8_t *utf8casestr(const utf8_int8_t *haystack, + const utf8_int8_t *needle) { + /* if needle has no utf8 codepoints before the null terminating + * byte then return haystack */ + if ('\0' == *needle) { + return (utf8_int8_t *)haystack; + } + + for (;;) { + const utf8_int8_t *maybeMatch = haystack; + const utf8_int8_t *n = needle; + utf8_int32_t h_cp = 0, n_cp = 0; + + /* Get the next code point and track it */ + const utf8_int8_t *nextH = haystack = utf8codepoint(haystack, &h_cp); + n = utf8codepoint(n, &n_cp); + + while ((0 != h_cp) && (0 != n_cp)) { + h_cp = utf8lwrcodepoint(h_cp); + n_cp = utf8lwrcodepoint(n_cp); + + /* if we find a mismatch, bail out! */ + if (h_cp != n_cp) { + break; + } + + haystack = utf8codepoint(haystack, &h_cp); + n = utf8codepoint(n, &n_cp); + } + + if (0 == n_cp) { + /* we found the whole utf8 string for needle in haystack at + * maybeMatch, so return it */ + return (utf8_int8_t *)maybeMatch; + } + + if (0 == h_cp) { + /* no match */ + return utf8_null; + } + + /* Roll back to the next code point in the haystack to test */ + haystack = nextH; + } +} + +utf8_constexpr14_impl utf8_int8_t *utf8valid(const utf8_int8_t *str) { + return utf8nvalid(str, SIZE_MAX); +} + +utf8_constexpr14_impl utf8_int8_t *utf8nvalid(const utf8_int8_t *str, + size_t n) { + const utf8_int8_t *t = str; + size_t consumed = 0; + + while ((void)(consumed = (size_t)(str - t)), consumed < n && '\0' != *str) { + const size_t remaining = n - consumed; + + if (0xf0 == (0xf8 & *str)) { + /* ensure that there's 4 bytes or more remaining */ + if (remaining < 4) { + return (utf8_int8_t *)str; + } + + /* ensure each of the 3 following bytes in this 4-byte + * utf8 codepoint began with 0b10xxxxxx */ + if ((0x80 != (0xc0 & str[1])) || (0x80 != (0xc0 & str[2])) || + (0x80 != (0xc0 & str[3]))) { + return (utf8_int8_t *)str; + } + + /* ensure that our utf8 codepoint ended after 4 bytes */ + if ((remaining != 4) && (0x80 == (0xc0 & str[4]))) { + return (utf8_int8_t *)str; + } + + /* ensure that the top 5 bits of this 4-byte utf8 + * codepoint were not 0, as then we could have used + * one of the smaller encodings */ + if ((0 == (0x07 & str[0])) && (0 == (0x30 & str[1]))) { + return (utf8_int8_t *)str; + } + + /* 4-byte utf8 code point (began with 0b11110xxx) */ + str += 4; + } else if (0xe0 == (0xf0 & *str)) { + /* ensure that there's 3 bytes or more remaining */ + if (remaining < 3) { + return (utf8_int8_t *)str; + } + + /* ensure each of the 2 following bytes in this 3-byte + * utf8 codepoint began with 0b10xxxxxx */ + if ((0x80 != (0xc0 & str[1])) || (0x80 != (0xc0 & str[2]))) { + return (utf8_int8_t *)str; + } + + /* ensure that our utf8 codepoint ended after 3 bytes */ + if ((remaining != 3) && (0x80 == (0xc0 & str[3]))) { + return (utf8_int8_t *)str; + } + + /* ensure that the top 5 bits of this 3-byte utf8 + * codepoint were not 0, as then we could have used + * one of the smaller encodings */ + if ((0 == (0x0f & str[0])) && (0 == (0x20 & str[1]))) { + return (utf8_int8_t *)str; + } + + /* 3-byte utf8 code point (began with 0b1110xxxx) */ + str += 3; + } else if (0xc0 == (0xe0 & *str)) { + /* ensure that there's 2 bytes or more remaining */ + if (remaining < 2) { + return (utf8_int8_t *)str; + } + + /* ensure the 1 following byte in this 2-byte + * utf8 codepoint began with 0b10xxxxxx */ + if (0x80 != (0xc0 & str[1])) { + return (utf8_int8_t *)str; + } + + /* ensure that our utf8 codepoint ended after 2 bytes */ + if ((remaining != 2) && (0x80 == (0xc0 & str[2]))) { + return (utf8_int8_t *)str; + } + + /* ensure that the top 4 bits of this 2-byte utf8 + * codepoint were not 0, as then we could have used + * one of the smaller encodings */ + if (0 == (0x1e & str[0])) { + return (utf8_int8_t *)str; + } + + /* 2-byte utf8 code point (began with 0b110xxxxx) */ + str += 2; + } else if (0x00 == (0x80 & *str)) { + /* 1-byte ascii (began with 0b0xxxxxxx) */ + str += 1; + } else { + /* we have an invalid 0b1xxxxxxx utf8 code point entry */ + return (utf8_int8_t *)str; + } + } + + return utf8_null; +} + +int utf8makevalid(utf8_int8_t *str, const utf8_int32_t replacement) { + utf8_int8_t *read = str; + utf8_int8_t *write = read; + const utf8_int8_t r = (utf8_int8_t)replacement; + utf8_int32_t codepoint = 0; + + if (replacement > 0x7f) { + return -1; + } + + while ('\0' != *read) { + if (0xf0 == (0xf8 & *read)) { + /* ensure each of the 3 following bytes in this 4-byte + * utf8 codepoint began with 0b10xxxxxx */ + if ((0x80 != (0xc0 & read[1])) || (0x80 != (0xc0 & read[2])) || + (0x80 != (0xc0 & read[3]))) { + *write++ = r; + read++; + continue; + } + + /* 4-byte utf8 code point (began with 0b11110xxx) */ + read = utf8codepoint(read, &codepoint); + write = utf8catcodepoint(write, codepoint, 4); + } else if (0xe0 == (0xf0 & *read)) { + /* ensure each of the 2 following bytes in this 3-byte + * utf8 codepoint began with 0b10xxxxxx */ + if ((0x80 != (0xc0 & read[1])) || (0x80 != (0xc0 & read[2]))) { + *write++ = r; + read++; + continue; + } + + /* 3-byte utf8 code point (began with 0b1110xxxx) */ + read = utf8codepoint(read, &codepoint); + write = utf8catcodepoint(write, codepoint, 3); + } else if (0xc0 == (0xe0 & *read)) { + /* ensure the 1 following byte in this 2-byte + * utf8 codepoint began with 0b10xxxxxx */ + if (0x80 != (0xc0 & read[1])) { + *write++ = r; + read++; + continue; + } + + /* 2-byte utf8 code point (began with 0b110xxxxx) */ + read = utf8codepoint(read, &codepoint); + write = utf8catcodepoint(write, codepoint, 2); + } else if (0x00 == (0x80 & *read)) { + /* 1-byte ascii (began with 0b0xxxxxxx) */ + read = utf8codepoint(read, &codepoint); + write = utf8catcodepoint(write, codepoint, 1); + } else { + /* if we got here then we've got a dangling continuation (0b10xxxxxx) */ + *write++ = r; + read++; + continue; + } + } + + *write = '\0'; + + return 0; +} + +utf8_constexpr14_impl utf8_int8_t * +utf8codepoint(const utf8_int8_t *utf8_restrict str, + utf8_int32_t *utf8_restrict out_codepoint) { + if (0xf0 == (0xf8 & str[0])) { + /* 4 byte utf8 codepoint */ + *out_codepoint = ((0x07 & str[0]) << 18) | ((0x3f & str[1]) << 12) | + ((0x3f & str[2]) << 6) | (0x3f & str[3]); + str += 4; + } else if (0xe0 == (0xf0 & str[0])) { + /* 3 byte utf8 codepoint */ + *out_codepoint = + ((0x0f & str[0]) << 12) | ((0x3f & str[1]) << 6) | (0x3f & str[2]); + str += 3; + } else if (0xc0 == (0xe0 & str[0])) { + /* 2 byte utf8 codepoint */ + *out_codepoint = ((0x1f & str[0]) << 6) | (0x3f & str[1]); + str += 2; + } else { + /* 1 byte utf8 codepoint otherwise */ + *out_codepoint = str[0]; + str += 1; + } + + return (utf8_int8_t *)str; +} + +utf8_constexpr14_impl size_t utf8codepointcalcsize(const utf8_int8_t *str) { + if (0xf0 == (0xf8 & str[0])) { + /* 4 byte utf8 codepoint */ + return 4; + } else if (0xe0 == (0xf0 & str[0])) { + /* 3 byte utf8 codepoint */ + return 3; + } else if (0xc0 == (0xe0 & str[0])) { + /* 2 byte utf8 codepoint */ + return 2; + } + + /* 1 byte utf8 codepoint otherwise */ + return 1; +} + +utf8_constexpr14_impl size_t utf8codepointsize(utf8_int32_t chr) { + if (0 == ((utf8_int32_t)0xffffff80 & chr)) { + return 1; + } else if (0 == ((utf8_int32_t)0xfffff800 & chr)) { + return 2; + } else if (0 == ((utf8_int32_t)0xffff0000 & chr)) { + return 3; + } else { /* if (0 == ((int)0xffe00000 & chr)) { */ + return 4; + } +} + +utf8_int8_t *utf8catcodepoint(utf8_int8_t *str, utf8_int32_t chr, size_t n) { + if (0 == ((utf8_int32_t)0xffffff80 & chr)) { + /* 1-byte/7-bit ascii + * (0b0xxxxxxx) */ + if (n < 1) { + return utf8_null; + } + str[0] = (utf8_int8_t)chr; + str += 1; + } else if (0 == ((utf8_int32_t)0xfffff800 & chr)) { + /* 2-byte/11-bit utf8 code point + * (0b110xxxxx 0b10xxxxxx) */ + if (n < 2) { + return utf8_null; + } + str[0] = (utf8_int8_t)(0xc0 | (utf8_int8_t)((chr >> 6) & 0x1f)); + str[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f)); + str += 2; + } else if (0 == ((utf8_int32_t)0xffff0000 & chr)) { + /* 3-byte/16-bit utf8 code point + * (0b1110xxxx 0b10xxxxxx 0b10xxxxxx) */ + if (n < 3) { + return utf8_null; + } + str[0] = (utf8_int8_t)(0xe0 | (utf8_int8_t)((chr >> 12) & 0x0f)); + str[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f)); + str[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f)); + str += 3; + } else { /* if (0 == ((int)0xffe00000 & chr)) { */ + /* 4-byte/21-bit utf8 code point + * (0b11110xxx 0b10xxxxxx 0b10xxxxxx 0b10xxxxxx) */ + if (n < 4) { + return utf8_null; + } + str[0] = (utf8_int8_t)(0xf0 | (utf8_int8_t)((chr >> 18) & 0x07)); + str[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 12) & 0x3f)); + str[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f)); + str[3] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f)); + str += 4; + } + + return str; +} + +utf8_constexpr14_impl int utf8islower(utf8_int32_t chr) { + return chr != utf8uprcodepoint(chr); +} + +utf8_constexpr14_impl int utf8isupper(utf8_int32_t chr) { + return chr != utf8lwrcodepoint(chr); +} + +void utf8lwr(utf8_int8_t *utf8_restrict str) { + utf8_int32_t cp = 0; + utf8_int8_t *pn = utf8codepoint(str, &cp); + + while (cp != 0) { + const utf8_int32_t lwr_cp = utf8lwrcodepoint(cp); + const size_t size = utf8codepointsize(lwr_cp); + + if (lwr_cp != cp) { + utf8catcodepoint(str, lwr_cp, size); + } + + str = pn; + pn = utf8codepoint(str, &cp); + } +} + +void utf8upr(utf8_int8_t *utf8_restrict str) { + utf8_int32_t cp = 0; + utf8_int8_t *pn = utf8codepoint(str, &cp); + + while (cp != 0) { + const utf8_int32_t lwr_cp = utf8uprcodepoint(cp); + const size_t size = utf8codepointsize(lwr_cp); + + if (lwr_cp != cp) { + utf8catcodepoint(str, lwr_cp, size); + } + + str = pn; + pn = utf8codepoint(str, &cp); + } +} + +utf8_constexpr14_impl utf8_int32_t utf8lwrcodepoint(utf8_int32_t cp) { + if (((0x0041 <= cp) && (0x005a >= cp)) || + ((0x00c0 <= cp) && (0x00d6 >= cp)) || + ((0x00d8 <= cp) && (0x00de >= cp)) || + ((0x0391 <= cp) && (0x03a1 >= cp)) || + ((0x03a3 <= cp) && (0x03ab >= cp)) || + ((0x0410 <= cp) && (0x042f >= cp))) { + cp += 32; + } else if ((0x0400 <= cp) && (0x040f >= cp)) { + cp += 80; + } else if (((0x0100 <= cp) && (0x012f >= cp)) || + ((0x0132 <= cp) && (0x0137 >= cp)) || + ((0x014a <= cp) && (0x0177 >= cp)) || + ((0x0182 <= cp) && (0x0185 >= cp)) || + ((0x01a0 <= cp) && (0x01a5 >= cp)) || + ((0x01de <= cp) && (0x01ef >= cp)) || + ((0x01f8 <= cp) && (0x021f >= cp)) || + ((0x0222 <= cp) && (0x0233 >= cp)) || + ((0x0246 <= cp) && (0x024f >= cp)) || + ((0x03d8 <= cp) && (0x03ef >= cp)) || + ((0x0460 <= cp) && (0x0481 >= cp)) || + ((0x048a <= cp) && (0x04ff >= cp))) { + cp |= 0x1; + } else if (((0x0139 <= cp) && (0x0148 >= cp)) || + ((0x0179 <= cp) && (0x017e >= cp)) || + ((0x01af <= cp) && (0x01b0 >= cp)) || + ((0x01b3 <= cp) && (0x01b6 >= cp)) || + ((0x01cd <= cp) && (0x01dc >= cp))) { + cp += 1; + cp &= ~0x1; + } else { + switch (cp) { + default: + break; + case 0x0178: + cp = 0x00ff; + break; + case 0x0243: + cp = 0x0180; + break; + case 0x018e: + cp = 0x01dd; + break; + case 0x023d: + cp = 0x019a; + break; + case 0x0220: + cp = 0x019e; + break; + case 0x01b7: + cp = 0x0292; + break; + case 0x01c4: + cp = 0x01c6; + break; + case 0x01c7: + cp = 0x01c9; + break; + case 0x01ca: + cp = 0x01cc; + break; + case 0x01f1: + cp = 0x01f3; + break; + case 0x01f7: + cp = 0x01bf; + break; + case 0x0187: + cp = 0x0188; + break; + case 0x018b: + cp = 0x018c; + break; + case 0x0191: + cp = 0x0192; + break; + case 0x0198: + cp = 0x0199; + break; + case 0x01a7: + cp = 0x01a8; + break; + case 0x01ac: + cp = 0x01ad; + break; + case 0x01b8: + cp = 0x01b9; + break; + case 0x01bc: + cp = 0x01bd; + break; + case 0x01f4: + cp = 0x01f5; + break; + case 0x023b: + cp = 0x023c; + break; + case 0x0241: + cp = 0x0242; + break; + case 0x03fd: + cp = 0x037b; + break; + case 0x03fe: + cp = 0x037c; + break; + case 0x03ff: + cp = 0x037d; + break; + case 0x037f: + cp = 0x03f3; + break; + case 0x0386: + cp = 0x03ac; + break; + case 0x0388: + cp = 0x03ad; + break; + case 0x0389: + cp = 0x03ae; + break; + case 0x038a: + cp = 0x03af; + break; + case 0x038c: + cp = 0x03cc; + break; + case 0x038e: + cp = 0x03cd; + break; + case 0x038f: + cp = 0x03ce; + break; + case 0x0370: + cp = 0x0371; + break; + case 0x0372: + cp = 0x0373; + break; + case 0x0376: + cp = 0x0377; + break; + case 0x03f4: + cp = 0x03b8; + break; + case 0x03cf: + cp = 0x03d7; + break; + case 0x03f9: + cp = 0x03f2; + break; + case 0x03f7: + cp = 0x03f8; + break; + case 0x03fa: + cp = 0x03fb; + break; + } + } + + return cp; +} + +utf8_constexpr14_impl utf8_int32_t utf8uprcodepoint(utf8_int32_t cp) { + if (((0x0061 <= cp) && (0x007a >= cp)) || + ((0x00e0 <= cp) && (0x00f6 >= cp)) || + ((0x00f8 <= cp) && (0x00fe >= cp)) || + ((0x03b1 <= cp) && (0x03c1 >= cp)) || + ((0x03c3 <= cp) && (0x03cb >= cp)) || + ((0x0430 <= cp) && (0x044f >= cp))) { + cp -= 32; + } else if ((0x0450 <= cp) && (0x045f >= cp)) { + cp -= 80; + } else if (((0x0100 <= cp) && (0x012f >= cp)) || + ((0x0132 <= cp) && (0x0137 >= cp)) || + ((0x014a <= cp) && (0x0177 >= cp)) || + ((0x0182 <= cp) && (0x0185 >= cp)) || + ((0x01a0 <= cp) && (0x01a5 >= cp)) || + ((0x01de <= cp) && (0x01ef >= cp)) || + ((0x01f8 <= cp) && (0x021f >= cp)) || + ((0x0222 <= cp) && (0x0233 >= cp)) || + ((0x0246 <= cp) && (0x024f >= cp)) || + ((0x03d8 <= cp) && (0x03ef >= cp)) || + ((0x0460 <= cp) && (0x0481 >= cp)) || + ((0x048a <= cp) && (0x04ff >= cp))) { + cp &= ~0x1; + } else if (((0x0139 <= cp) && (0x0148 >= cp)) || + ((0x0179 <= cp) && (0x017e >= cp)) || + ((0x01af <= cp) && (0x01b0 >= cp)) || + ((0x01b3 <= cp) && (0x01b6 >= cp)) || + ((0x01cd <= cp) && (0x01dc >= cp))) { + cp -= 1; + cp |= 0x1; + } else { + switch (cp) { + default: + break; + case 0x00ff: + cp = 0x0178; + break; + case 0x0180: + cp = 0x0243; + break; + case 0x01dd: + cp = 0x018e; + break; + case 0x019a: + cp = 0x023d; + break; + case 0x019e: + cp = 0x0220; + break; + case 0x0292: + cp = 0x01b7; + break; + case 0x01c6: + cp = 0x01c4; + break; + case 0x01c9: + cp = 0x01c7; + break; + case 0x01cc: + cp = 0x01ca; + break; + case 0x01f3: + cp = 0x01f1; + break; + case 0x01bf: + cp = 0x01f7; + break; + case 0x0188: + cp = 0x0187; + break; + case 0x018c: + cp = 0x018b; + break; + case 0x0192: + cp = 0x0191; + break; + case 0x0199: + cp = 0x0198; + break; + case 0x01a8: + cp = 0x01a7; + break; + case 0x01ad: + cp = 0x01ac; + break; + case 0x01b9: + cp = 0x01b8; + break; + case 0x01bd: + cp = 0x01bc; + break; + case 0x01f5: + cp = 0x01f4; + break; + case 0x023c: + cp = 0x023b; + break; + case 0x0242: + cp = 0x0241; + break; + case 0x037b: + cp = 0x03fd; + break; + case 0x037c: + cp = 0x03fe; + break; + case 0x037d: + cp = 0x03ff; + break; + case 0x03f3: + cp = 0x037f; + break; + case 0x03ac: + cp = 0x0386; + break; + case 0x03ad: + cp = 0x0388; + break; + case 0x03ae: + cp = 0x0389; + break; + case 0x03af: + cp = 0x038a; + break; + case 0x03cc: + cp = 0x038c; + break; + case 0x03cd: + cp = 0x038e; + break; + case 0x03ce: + cp = 0x038f; + break; + case 0x0371: + cp = 0x0370; + break; + case 0x0373: + cp = 0x0372; + break; + case 0x0377: + cp = 0x0376; + break; + case 0x03d1: + cp = 0x0398; + break; + case 0x03d7: + cp = 0x03cf; + break; + case 0x03f2: + cp = 0x03f9; + break; + case 0x03f8: + cp = 0x03f7; + break; + case 0x03fb: + cp = 0x03fa; + break; + } + } + + return cp; +} + +utf8_constexpr14_impl utf8_int8_t * +utf8rcodepoint(const utf8_int8_t *utf8_restrict str, + utf8_int32_t *utf8_restrict out_codepoint) { + const utf8_int8_t *s = (const utf8_int8_t *)str; + + if (0xf0 == (0xf8 & s[0])) { + /* 4 byte utf8 codepoint */ + *out_codepoint = ((0x07 & s[0]) << 18) | ((0x3f & s[1]) << 12) | + ((0x3f & s[2]) << 6) | (0x3f & s[3]); + } else if (0xe0 == (0xf0 & s[0])) { + /* 3 byte utf8 codepoint */ + *out_codepoint = + ((0x0f & s[0]) << 12) | ((0x3f & s[1]) << 6) | (0x3f & s[2]); + } else if (0xc0 == (0xe0 & s[0])) { + /* 2 byte utf8 codepoint */ + *out_codepoint = ((0x1f & s[0]) << 6) | (0x3f & s[1]); + } else { + /* 1 byte utf8 codepoint otherwise */ + *out_codepoint = s[0]; + } + + do { + s--; + } while ((0 != (0x80 & s[0])) && (0x80 == (0xc0 & s[0]))); + + return (utf8_int8_t *)s; +} + +#undef utf8_restrict +#undef utf8_constexpr14 +#undef utf8_null + +#ifdef utf8_cplusplus +} /* extern "C" */ +#endif + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +#endif /* SHEREDOM_UTF8_H_INCLUDED */ diff --git a/src/vm/vm.c b/src/vm/vm.c index 1dc07fb95..3caef3d8d 100644 --- a/src/vm/vm.c +++ b/src/vm/vm.c @@ -10,6 +10,7 @@ #include "object.h" #include "memory.h" #include "vm.h" +#include "utf8.h" #include "util.h" #include "error_lib/error.h" #include "datatypes/number.h" @@ -1898,14 +1899,26 @@ static DictuInterpretResult run(DictuVM *vm) { case OBJ_STRING: { ObjString *string = AS_STRING(subscriptValue); + int len = string->character_len == -1 ? string->length : string->character_len; int index = AS_NUMBER(indexValue); // Allow negative indexes if (index < 0) - index = string->length + index; - - if (index >= 0 && index < string->length) { - ObjString *newString = copyString(vm, &string->chars[index], 1); + index = len + index; + + if (index >= 0 && index < len) { + ObjString* newString; + if(string->character_len != -1 && string->character_len != string->length) { + utf8_int32_t ch; + char* ptr = string->chars; + for(size_t i = 0; i <= (size_t)index; i++) + ptr = utf8codepoint(ptr, &ch); + size_t cpSize = utf8codepointsize(ch); + newString = copyString(vm, ptr - cpSize, cpSize); + + } else { + newString = copyString(vm, &string->chars[index], 1); + } pop(vm); pop(vm); push(vm, OBJ_VAL(newString)); @@ -2103,16 +2116,17 @@ static DictuInterpretResult run(DictuVM *vm) { case OBJ_STRING: { ObjString *string = AS_STRING(objectValue); + int len = string->character_len == -1 ? string->length : string->character_len; if (IS_EMPTY(sliceEndIndex)) { - indexEnd = string->length; + indexEnd = len; } else { indexEnd = AS_NUMBER(sliceEndIndex); - if (indexEnd > string->length) { - indexEnd = string->length; + if (indexEnd > len) { + indexEnd = len; } else if (indexEnd < 0) { - indexEnd = string->length + indexEnd; + indexEnd = len + indexEnd; } } @@ -2120,7 +2134,20 @@ static DictuInterpretResult run(DictuVM *vm) { if (indexStart > indexEnd) { returnVal = OBJ_VAL(copyString(vm, "", 0)); } else { - returnVal = OBJ_VAL(copyString(vm, string->chars + indexStart, indexEnd - indexStart)); + if(string->character_len != -1 && string->character_len != string->length) { + utf8_int32_t ch; + char* ptr = string->chars; + // first we need to advance by the skip; + for(size_t i = 0; i < (size_t)indexStart; i++) + ptr = utf8codepoint(ptr, &ch); + char* ptrEnd = ptr; + for(size_t i = 0; i < (size_t)(indexEnd - indexStart); i++) + ptrEnd = utf8codepoint(ptrEnd, &ch); + returnVal = OBJ_VAL(copyString(vm, ptr, ptrEnd - ptr)); + } else { + returnVal = OBJ_VAL(copyString(vm, string->chars + indexStart, indexEnd - indexStart)); + } + } break; } diff --git a/tests/strings/collapseSpaces.du b/tests/strings/collapseSpaces.du index d885ee3ff..766b5f42d 100644 --- a/tests/strings/collapseSpaces.du +++ b/tests/strings/collapseSpaces.du @@ -15,6 +15,15 @@ class TestStringCollapseSpaces < UnitTest { this.assertEquals(res, expected); this.assertNotEquals(testString, expected); } + + testStringCollapseSpacesUnicode() { + const testString = "This is 🌍 a huge string🐦 🐦 of a lot of spaces."; + const expected = "This is 🌍 a huge string🐦 🐦 of a lot of spaces."; + const res = testString.collapseSpaces(); + this.assertEquals(res, expected); + this.assertNotEquals(testString, expected); + } + } TestStringCollapseSpaces().run(); diff --git a/tests/strings/concat.du b/tests/strings/concat.du index 9518698ca..fed763bf8 100644 --- a/tests/strings/concat.du +++ b/tests/strings/concat.du @@ -15,6 +15,16 @@ class TestStringConcat < UnitTest { this.assertEquals(x, "Dictu is great!"); } + testStringConcatUnicode() { + this.assertEquals("πŸ˜…" + " πŸ˜… " + "πŸ˜…", "πŸ˜… πŸ˜… πŸ˜…"); + + const x = "Dictu" + + " 😊 " + + "great❗"; + + this.assertEquals(x, "Dictu 😊 great❗"); + } + } TestStringConcat().run(); diff --git a/tests/strings/contains.du b/tests/strings/contains.du index 557cc0b92..48b9eabd8 100644 --- a/tests/strings/contains.du +++ b/tests/strings/contains.du @@ -17,6 +17,12 @@ class TestStringContains < UnitTest { this.assertTruthy("1Dictu is great!1".contains("1")); this.assertTruthy(("1Dictu " + "is great!").contains("1")); } + testStringContainsUnicode() { + this.assertTruthy(("❗Dictu " + "is great!").contains("❗")); + this.assertFalsey("Dictu is great!".contains("❗")); + this.assertTruthy("Dictu is ♨️ right?️".contains("♨️")); // ♨️ is two characters! + + } } TestStringContains().run(); \ No newline at end of file diff --git a/tests/strings/count.du b/tests/strings/count.du index 873e57147..210ca3f80 100644 --- a/tests/strings/count.du +++ b/tests/strings/count.du @@ -16,6 +16,12 @@ class TestStringCount < UnitTest { this.assertEquals("Dictu is great! Dictu is great!".count("1234"), 0); this.assertEquals("Dictu is great! Dictu is great!".count("!"), 2); } + testStringCountUnicode() { + this.assertEquals("Dictu is 😎! Dictu is 😎!".count("Dictu is 😎!"), 2); + this.assertEquals("Dictu is great❗".count("❗"), 1); + this.assertEquals("Dictu is great❕ Dictu is great❕".count("❗"), 0); + + } } TestStringCount().run(); \ No newline at end of file diff --git a/tests/strings/endsWith.du b/tests/strings/endsWith.du index 5447b9d64..6f8085ac8 100644 --- a/tests/strings/endsWith.du +++ b/tests/strings/endsWith.du @@ -19,6 +19,13 @@ class TestStringEndsWith < UnitTest { this.assertFalsey("test".endsWith("!@Β£$%")); this.assertFalsey("test".endsWith("123456789")); } + testStringEndsWithUnicode() { + this.assertTruthy("πŸŒπŸ˜βš“πŸŒ".endsWith("🌍")); + this.assertTruthy("πŸŒπŸ˜βš“πŸŒ".endsWith("βš“πŸŒ")); + this.assertTruthy("πŸŒπŸ˜βš“πŸŒ".endsWith("πŸŒπŸ˜βš“πŸŒ")); + this.assertFalsey("πŸŒπŸ˜βš“πŸŒ".endsWith("πŸ˜…")); + this.assertFalsey("πŸŒπŸ˜βš“πŸŒ".endsWith("πŸŒπŸ˜βš“πŸŒπŸ˜…")); + } } TestStringEndsWith().run(); \ No newline at end of file diff --git a/tests/strings/find.du b/tests/strings/find.du index 2ba1def7b..59ec6af2c 100644 --- a/tests/strings/find.du +++ b/tests/strings/find.du @@ -21,7 +21,12 @@ class TestStringFind < UnitTest { } testStringFindThird() { - this.assertEquals("Dictu is great! Dictu is great! Dictu is great!".find("Dictu", 3), 37); + this.assertEquals("Dictu is great! Dictu is great! Dictu is great!".find("Dictu", 3), 32); + } + testStringFindUnicode() { + this.assertEquals("πŸ’»πŸ˜ƒπŸ˜ƒπŸ˜ƒa".find("πŸ’»"), 0); + this.assertEquals("πŸ’»πŸ˜ƒπŸ˜ƒπŸ˜ƒa".find("πŸ˜ƒ",2), 2); + this.assertEquals("πŸ’»πŸ˜ƒπŸ˜ƒπŸ˜ƒa".find("πŸ’»",2), -1); } testStringFindDoesntExist() { diff --git a/tests/strings/findLast.du b/tests/strings/findLast.du index 1f7d04ea1..8eed13451 100644 --- a/tests/strings/findLast.du +++ b/tests/strings/findLast.du @@ -12,6 +12,11 @@ class TestStringLastIndexOf < UnitTest { this.assertEquals("woolly woolly mammoth".findLast("woolly"), 7); this.assertEquals("mammoth".findLast("woolly"), -1); } + testStringLowerUnicode() { + this.assertEquals("πŸŒπŸ˜βš“πŸŒ πŸŒπŸ˜βš“πŸŒ mammoth".findLast("πŸŒπŸ˜βš“πŸŒ"), 5); + this.assertEquals("πŸŒπŸ˜βš“πŸŒ".findLast("πŸ˜…πŸ˜…"), -1); + + } } TestStringLastIndexOf().run(); \ No newline at end of file diff --git a/tests/strings/format.du b/tests/strings/format.du index 4d186c542..7375e1d43 100644 --- a/tests/strings/format.du +++ b/tests/strings/format.du @@ -21,6 +21,10 @@ class TestStringFormat < UnitTest { this.assertEquals("{}".format({"test": 10, "aaa": 10}), '{"aaa": 10, "test": 10}'); this.assertEquals("{} {} {}".format(test, Test, Trait), ' '); } + testStringFormatUnicode() { + this.assertEquals("World: {} is {}".format("🌍", "😎"), "World: 🌍 is 😎"); + this.assertEquals("hello {}".format(["πŸŒπŸ˜βš“πŸŒ"]), "hello [\"πŸŒπŸ˜βš“πŸŒ\"]"); + } } TestStringFormat().run(); \ No newline at end of file diff --git a/tests/strings/isLower.du b/tests/strings/isLower.du index 5b94fd345..324d7ce6e 100644 --- a/tests/strings/isLower.du +++ b/tests/strings/isLower.du @@ -16,6 +16,13 @@ class TestStringIsLower < UnitTest { this.assertFalsey("D".isLower()); this.assertFalsey("Maple".isLower()); } + testStringIsLowerUnicode() { + this.assertTruthy("dΓ€".isLower()); + this.assertTruthy("dΓΆgog".isLower()); + this.assertTruthy("g0Γ€0d!".isLower()); + this.assertFalsey("Γ„".isLower()); + this.assertFalsey("Ma😁ple".isLower()); + } } TestStringIsLower().run(); diff --git a/tests/strings/isUpper.du b/tests/strings/isUpper.du index f7d3294cb..c6444473f 100644 --- a/tests/strings/isUpper.du +++ b/tests/strings/isUpper.du @@ -15,6 +15,12 @@ class TestStringIsUpper < UnitTest { this.assertTruthy("G00D!".isUpper()); this.assertFalsey("Maple".isUpper()); } + testStringIsUpperUnicode() { + this.assertTruthy("πŸ˜†D".isUpper()); + this.assertTruthy("DOπŸ˜†G".isUpper()); + this.assertTruthy("GπŸ˜†00D!".isUpper()); + this.assertFalsey("MaπŸ˜†ple".isUpper()); + } } TestStringIsUpper().run(); diff --git a/tests/strings/len.du b/tests/strings/len.du index c8fdca006..73f0a864f 100644 --- a/tests/strings/len.du +++ b/tests/strings/len.du @@ -11,9 +11,12 @@ class TestStringLen < UnitTest { testStringLen() { this.assertType("qwert".len(), "number"); this.assertEquals("qwert".len(), 5); - this.assertEquals("ασδφγ".len(), 10); + this.assertEquals("ασδφγ".len(), 5); this.assertEquals(('test' + 'ing').len(), 7); } + testStringLenUnicode() { + this.assertEquals("πŸ€ͺ".len(), 1); + } } TestStringLen().run(); \ No newline at end of file diff --git a/tests/strings/lower.du b/tests/strings/lower.du index 9f7f58857..3fd091d5e 100644 --- a/tests/strings/lower.du +++ b/tests/strings/lower.du @@ -16,6 +16,10 @@ class TestStringLower < UnitTest { this.assertEquals("12Dictu45".lower(), "12dictu45"); this.assertEquals("!@Β£$%^&*".lower(), "!@Β£$%^&*"); } + testStringLowerUnicode() { + this.assertEquals("Γ„ΓœΓ–".lower(), "Àüâ"); + this.assertEquals("DICTU🌍".lower(), "dictu🌍"); + } } TestStringLower().run(); \ No newline at end of file diff --git a/tests/strings/repeat.du b/tests/strings/repeat.du index 60b6f8462..59bbdfcd9 100644 --- a/tests/strings/repeat.du +++ b/tests/strings/repeat.du @@ -12,6 +12,11 @@ class TestStringRepeat < UnitTest { this.assertEquals("ba" + "na".repeat(2), "banana"); this.assertEquals("ha".repeat(6), "hahahahahaha"); } + testStringRepeatUnicode() { + this.assertEquals("βŒ›" + "⏳".repeat(2), "βŒ›β³β³"); + this.assertEquals("πŸ“ƒ".repeat(5), "πŸ“ƒπŸ“ƒπŸ“ƒπŸ“ƒπŸ“ƒ"); + } + } TestStringRepeat().run(); diff --git a/tests/strings/replace.du b/tests/strings/replace.du index e8a215e5e..d00598e61 100644 --- a/tests/strings/replace.du +++ b/tests/strings/replace.du @@ -21,6 +21,11 @@ class TestStringReplace < UnitTest { this.assertEquals("test".replace("t", "123456789123456789123456789123456789"), "123456789123456789123456789123456789es123456789123456789123456789123456789"); } + testStringReplaceUnicode() { + this.assertEquals("πŸ˜€".replace("πŸ˜€", "πŸ˜…"), "πŸ˜…"); + this.assertEquals("πŸ˜€πŸ˜€".replace("πŸ˜€", "πŸ˜€πŸ˜€"), "πŸ˜€πŸ˜€πŸ˜€πŸ˜€"); + this.assertEquals("❕string❗".replace("❗", "❕❕"), "❕string❕❕"); + } } TestStringReplace().run(); \ No newline at end of file diff --git a/tests/strings/slicing.du b/tests/strings/slicing.du index 717119a8f..2d8d2611c 100644 --- a/tests/strings/slicing.du +++ b/tests/strings/slicing.du @@ -34,6 +34,33 @@ class TestStringSlicing < UnitTest { this.assertEquals(x[2:4], "ct"); this.assertEquals(x[2:3], "c"); } + testStringSliceUnicode() { + const x = "πŸ˜€πŸŒπŸ¦aΓ€"; + this.assertEquals(x[0:x.len()], x); + this.assertEquals(x[0:], x); + this.assertEquals(x[1:], "🌍🐦aΓ€"); + this.assertEquals(x[2:], "🐦aΓ€"); + this.assertEquals(x[3:], "aΓ€"); + this.assertEquals(x[4:], "Γ€"); + this.assertEquals(x[5:], ""); + + this.assertEquals(x[:5], "πŸ˜€πŸŒπŸ¦aΓ€"); + this.assertEquals(x[:4], "πŸ˜€πŸŒπŸ¦a"); + this.assertEquals(x[:3], "πŸ˜€πŸŒπŸ¦"); + this.assertEquals(x[:2], "πŸ˜€πŸŒ"); + this.assertEquals(x[:1], "πŸ˜€"); + this.assertEquals(x[:0], ""); + + this.assertEquals(x[1:4], "🌍🐦a"); + this.assertEquals(x[1:3], "🌍🐦"); + this.assertEquals(x[1:2], "🌍"); + this.assertEquals(x[1:1], ""); + + this.assertEquals(x[2:5], "🐦aΓ€"); + this.assertEquals(x[2:4], "🐦a"); + this.assertEquals(x[2:3], "🐦"); + this.assertEquals(x[:x.len()], x); + } } TestStringSlicing().run(); \ No newline at end of file diff --git a/tests/strings/split.du b/tests/strings/split.du index 165e2d76e..def59114e 100644 --- a/tests/strings/split.du +++ b/tests/strings/split.du @@ -28,6 +28,12 @@ class TestStringSplit < UnitTest { this.assertEquals("Dictu is great!".split(" ", 3), ["Dictu", "is", "great!"]); this.assertEquals("Dictu is great!".split(" ", 4), ["Dictu", "is", "great!"]); } + + testStringSplitUnicode() { + + this.assertEquals("DictuπŸ’»isπŸ’»veryπŸ’»πŸ˜ƒCoolπŸ˜ƒ".split("πŸ’»"), ["Dictu", "is", "very", "πŸ˜ƒCoolπŸ˜ƒ"]); + this.assertEquals("Dictu is veryπŸ˜ƒCoolπŸ˜ƒ!".split("πŸ˜ƒCoolπŸ˜ƒ"), ["Dictu is very", "!"]); + } testStringSplitOptionalSinglular() { this.assertEquals("Dictu is great!".split("", -2), ["D", "i", "c", "t", "u", " ", "i", "s", " ", "g", "r", "e", "a", "t", "!"]); diff --git a/tests/strings/startsWith.du b/tests/strings/startsWith.du index c5b8272d1..d74d39809 100644 --- a/tests/strings/startsWith.du +++ b/tests/strings/startsWith.du @@ -19,6 +19,11 @@ class TestStringStartsWith < UnitTest { this.assertFalsey("test".startsWith("!@Β£$%")); this.assertFalsey("test".startsWith("123456789")); } + testStringStartsWithUnicode() { + this.assertTruthy("πŸ’»test".startsWith("πŸ’»t")); + this.assertTruthy("πŸ’»test".startsWith("πŸ’»test")); + this.assertFalsey("πŸ’»test".startsWith("πŸ’»teπŸ’»")); + } } TestStringStartsWith().run(); \ No newline at end of file diff --git a/tests/strings/strip.du b/tests/strings/strip.du index 3297a1a47..3880d61ec 100644 --- a/tests/strings/strip.du +++ b/tests/strings/strip.du @@ -27,7 +27,7 @@ class TestStringStrip < UnitTest { this.assertEquals(" Dictu is great!".rightStrip().len(), 17); this.assertEquals(" Dictu is great! ".rightStrip().len(), 17); } - + testStringStrip() { this.assertEquals("Dictu is great! ".strip(), "Dictu is great!"); this.assertEquals(" Dictu is great!".strip(), "Dictu is great!"); @@ -36,6 +36,17 @@ class TestStringStrip < UnitTest { this.assertEquals(" Dictu is great!".strip().len(), 15); this.assertEquals(" Dictu is great! ".strip().len(), 15); } + + testStringStripUnicode() { + this.assertEquals("🐦Dictu is great🐦 ".strip(), "🐦Dictu is great🐦"); + this.assertEquals(" 🐦Dictu is great!🐦".strip(), "🐦Dictu is great!🐦"); + this.assertEquals(" 🐦Dictu is great!🐦 ".strip(), "🐦Dictu is great!🐦"); + this.assertEquals("Dictu is great🐦 ".strip().len(), 15); + this.assertEquals(" Dictu is great🐦".strip().len(), 15); + this.assertEquals(" 🐦Dictu is great🐦 ".strip().len(), 16); + } + + } TestStringStrip().run(); \ No newline at end of file diff --git a/tests/strings/subscript.du b/tests/strings/subscript.du index eae3a39dc..f189a3345 100644 --- a/tests/strings/subscript.du +++ b/tests/strings/subscript.du @@ -18,6 +18,19 @@ class TestStringSubscript < UnitTest { this.assertEquals(string[-2], "t"); this.assertEquals(string[-3], "a"); + for (var i = 0; i < string.len(); i += 1) { + this.assertEquals(string[i], stringList[i]); + } + } + testStringSubscriptUnicode() { + const string = "πŸ˜€πŸŒπŸ¦aΓ€"; + const stringList = ["πŸ˜€", "🌍", "🐦", "a", "Γ€"]; + + this.assertEquals(string[0], "πŸ˜€"); + this.assertEquals(string[1], "🌍"); + this.assertEquals(string[2], "🐦"); + this.assertEquals(string[-1], "Γ€"); + for (var i = 0; i < string.len(); i += 1) { this.assertEquals(string[i], stringList[i]); } diff --git a/tests/strings/title.du b/tests/strings/title.du index ecd1c85e1..b17bd47a9 100644 --- a/tests/strings/title.du +++ b/tests/strings/title.du @@ -17,6 +17,13 @@ class TestStringTitle < UnitTest { this.assertEquals("!@Β£$%^&*".title(), "!@Β£$%^&*"); this.assertEquals("once upon a time".title(), "Once Upon A Time"); } + testStringTitleUnicode() { + this.assertEquals("ΓΌber".title(), "Über"); + this.assertEquals("ΓΌBer".title(), "Über"); + this.assertEquals("πŸ‘»πŸ‘»".title(), "πŸ‘»πŸ‘»"); + this.assertEquals("πŸ‘»πŸ‘» test".title(), "πŸ‘»πŸ‘» Test"); + this.assertEquals("πŸ‘»πŸ‘» test ΓΌBer".title(), "πŸ‘»πŸ‘» Test Über"); + } } TestStringTitle().run(); \ No newline at end of file diff --git a/tests/strings/toBool.du b/tests/strings/toBool.du index 719b035f6..5c6f8928d 100644 --- a/tests/strings/toBool.du +++ b/tests/strings/toBool.du @@ -12,6 +12,9 @@ class TestStringToBool < UnitTest { this.assertTruthy("10".toBool()); this.assertFalsey("".toBool()); } + testStringToBoolUnicode() { + this.assertTruthy("πŸ’¬".toBool()); + } } TestStringToBool().run(); \ No newline at end of file diff --git a/tests/strings/upper.du b/tests/strings/upper.du index a7012e976..40a444092 100644 --- a/tests/strings/upper.du +++ b/tests/strings/upper.du @@ -16,6 +16,10 @@ class TestStringUpper < UnitTest { this.assertEquals("12Dictu45".upper(), "12DICTU45"); this.assertEquals("!@Β£$%^&*".upper(), "!@Β£$%^&*"); } + testStringUpperUnicode() { + this.assertEquals("üÀâ".upper(), "ΓœΓ„Γ–"); + this.assertEquals("ü🌐test".upper(), "Ü🌐TEST"); + } } TestStringUpper().run(); \ No newline at end of file diff --git a/tests/strings/wordCount.du b/tests/strings/wordCount.du index 6f93cfde1..614321f8b 100644 --- a/tests/strings/wordCount.du +++ b/tests/strings/wordCount.du @@ -14,6 +14,10 @@ class TestStringWordCount < UnitTest { this.assertEquals("This is a sentence".wordCount(), 4); this.assertEquals("This is an even longer sentence".wordCount(), 6); } + testStringWordCountUnicode() { + this.assertEquals("Dictu is cool😎".wordCount(), 3); + this.assertEquals("WOrld🌐 üÀâ".wordCount(), 2); + } } TestStringWordCount().run(); diff --git a/tests/strings/wrap.du b/tests/strings/wrap.du index 442b04652..0de75e361 100644 --- a/tests/strings/wrap.du +++ b/tests/strings/wrap.du @@ -16,6 +16,11 @@ class TestStringWrap < UnitTest { const idx = res.find("\n"); this.assertTruthy(res.find("\n") <= this.maxLen); } + testStringWrapUnicode() { + const _len = 5; + const wrapped = "🌐test 🐦🐦🐦 test344🐦".wrap(_len); + this.assertEquals(wrapped, "🌐test\n🐦🐦🐦\ntest344🐦"); + } } TestStringWrap().run(); diff --git a/tests/uuid/generate.du b/tests/uuid/generate.du index b2297b9e8..65c0e41c4 100644 --- a/tests/uuid/generate.du +++ b/tests/uuid/generate.du @@ -16,7 +16,7 @@ class TestUUIDGenerate < UnitTest { this.assertTruthy(ret.success()); const uuid = ret.unwrap(); this.assertType(type(uuid), "string"); - this.assertTruthy(uuid.len() == 37); + this.assertTruthy(uuid.byteLen() == 36); print(uuid); } } diff --git a/tests/uuid/generateRandom.du b/tests/uuid/generateRandom.du index 0a66edcb8..f1844627d 100644 --- a/tests/uuid/generateRandom.du +++ b/tests/uuid/generateRandom.du @@ -16,7 +16,7 @@ class TestUUIDGenerateRandom < UnitTest { this.assertTruthy(ret.success()); const uuid = ret.unwrap(); this.assertType(type(uuid), "string"); - this.assertTruthy(uuid.len() == 37); + this.assertTruthy(uuid.byteLen() == 36); } } diff --git a/tests/uuid/generateTime.du b/tests/uuid/generateTime.du index efe051669..2f88e7810 100644 --- a/tests/uuid/generateTime.du +++ b/tests/uuid/generateTime.du @@ -16,7 +16,7 @@ class TestUUIDGenerateTime < UnitTest { this.assertTruthy(ret.success()); const uuid = ret.unwrap(); this.assertType(type(uuid), "string"); - this.assertTruthy(uuid.len() == 37); + this.assertTruthy(uuid.byteLen() == 36); } }