From 31d0d4f2f47fa93961d29f5d93887c6367f877e2 Mon Sep 17 00:00:00 2001 From: Amit Gupta Date: Sun, 29 Dec 2024 14:58:19 -0600 Subject: [PATCH 01/14] Implement kim_base64_encode A utility for base64 encoding and decoding of model and model-driver parameter files. This biary executable will be used for compiling shared-libraries of model and model-driver. This utility will be installed in the CMAKE_INSTALL_RELOC_BINDIR. --- CMakeLists.txt | 5 + cmake/items-macros.cmake.in | 17 +-- cpp/include/base64.hpp | 191 ++++++++++++++++++++++++++++++++++ cpp/src/KIM_SharedLibrary.cpp | 52 ++++++++- utils/encode.cpp | 71 +++++++++++++ 5 files changed, 326 insertions(+), 10 deletions(-) create mode 100644 cpp/include/base64.hpp create mode 100644 utils/encode.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3d369378..26b5a05e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,12 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug") set(_LOG_MAX "DEBUG") else() set(_LOG_MAX "INFORMATION") + set(CMAKE_BUILD_TYPE "Release") # Default to Release, soves KIM slowdown complaints endif() + +# C++ Language Standard, enforced to be C++11 +set(CMAKE_CXX_STANDARD 11) + set_cache_with_fallback(KIM_API_LOG_MAXIMUM_LEVEL "${_LOG_MAX}" STRING "Maximum log verbosity") unset(_LOG_MAX) set_property(CACHE KIM_API_LOG_MAXIMUM_LEVEL PROPERTY STRINGS "" SILENT FATAL ERROR WARNING INFORMATION DEBUG) diff --git a/cmake/items-macros.cmake.in b/cmake/items-macros.cmake.in index 78d33e9e..b2701553 100644 --- a/cmake/items-macros.cmake.in +++ b/cmake/items-macros.cmake.in @@ -494,18 +494,19 @@ macro(_set_simulator_model_template_variables) endmacro() # -# Function to create a custom command to convert a file (blob) to a c source. +# Function to create a custom command to convert a file (blob) to a cpp source. # # Convert _filein to hex then format as c source file and write to _fileout. # Assumes _filein is a relative file name in the _dirin directory. # # Sets _blob_to_c_command to string for use with add_custom_command() # -function(_add_custom_command_blob_to_c _dirin _filein _fileout) +function(_add_custom_command_blob_to_cpp _dirin _filein _fileout) include(FindPackageMessage) - find_program(XXD_EXECUTABLE "xxd") + # find_program(XXD_EXECUTABLE "xxd") + find_program(XXD_EXECUTABLE "kim_base64_encode") if(XXD_EXECUTABLE) - find_package_message(xxd "Found xxd: (${XXD_EXECUTABLE})" "found") + find_package_message(xxd "Found kim_base64_encode: (${XXD_EXECUTABLE})" "found") string(MAKE_C_IDENTIFIER ${_filein} _cfilein) set(_edit_xxd_output "${CMAKE_CURRENT_BINARY_DIR}/EditXXDOutput.cmake") @@ -522,7 +523,7 @@ function(_add_custom_command_blob_to_c _dirin _filein _fileout) ) endif() add_custom_command(OUTPUT ${_fileout} - COMMAND ${XXD_EXECUTABLE} -i "${_filein}" "${_fileout}" + COMMAND ${XXD_EXECUTABLE} "${_filein}" "${_fileout}" COMMAND ${CMAKE_COMMAND} "-Dvarname=${_cfilein}" "-Dfilein=${_filein}" "-Dfileout=${_fileout}" -P "${_edit_xxd_output}" DEPENDS "${_dirin}/${_filein}" "${_edit_xxd_output}" WORKING_DIRECTORY "${_dirin}" @@ -555,7 +556,7 @@ function(_add_custom_command_blob_to_c _dirin _filein _fileout) unset(_cfilein) unset(_blob_to_source) endif() -endfunction(_add_custom_command_blob_to_c) +endfunction(_add_custom_command_blob_to_cpp) # # Function to create and define targets for c source files from provided data @@ -607,9 +608,9 @@ function(_xxd_process_files) else() message(FATAL_ERROR "Cannot find metadata file '${_FNAME}'") endif() - set(_FILE "${CMAKE_CURRENT_BINARY_DIR}/${_FNAME}.c") + set(_FILE "${CMAKE_CURRENT_BINARY_DIR}/${_FNAME}.cpp") list(APPEND _ITEM_SOURCES ${_FILE}) - _add_custom_command_blob_to_c("${_DIR_IN}" "${_FNAME}" "${_FILE}") + _add_custom_command_blob_to_cpp("${_DIR_IN}" "${_FNAME}" "${_FILE}") unset(_DIR_IN) endforeach() unset(_FNAME) diff --git a/cpp/include/base64.hpp b/cpp/include/base64.hpp new file mode 100644 index 00000000..e2cc91f1 --- /dev/null +++ b/cpp/include/base64.hpp @@ -0,0 +1,191 @@ +#ifndef BASE64_HPP +#define BASE64_HPP + +#include +#include +#include + +class Base64 { +// Based on boost https://www.boost.org/doc/libs/1_66_0/boost/beast/core/detail/base64.hpp +private: + // Base64 alphabet table + inline static const char* get_alphabet() { + static const char tab[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + return tab; + } + + // Base64 inverse lookup table + inline static const signed char* get_inverse() { + static const signed char tab[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 + }; + return tab; + } + + +public: + // Prevent instantiation + Base64() = delete; + // Calculate encoded size + inline static constexpr size_t encoded_size(size_t n) { + return 4 * ((n + 2) / 3); + } + + // Calculate decoded size + inline static constexpr size_t decoded_size(size_t n) { + return n / 4 * 3; + } + + constexpr static std::size_t MAX_BASE64_WIDTH = 76; + constexpr static std::size_t MAX_BINARY_WIDTH = MAX_BASE64_WIDTH/4 * 3; + + + + inline static std::string encode(const std::string& input) { + std::string output; + output.resize(encoded_size(input.size())); + + const char* in = input.data(); + char* out = &output[0]; + size_t len = input.size(); + auto const tab = get_alphabet(); + + // Process 3-byte chunks + for(size_t n = len / 3; n--;) { + *out++ = tab[(in[0] & 0xfc) >> 2]; + *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; + *out++ = tab[((in[2] & 0xc0) >> 6) + ((in[1] & 0x0f) << 2)]; + *out++ = tab[in[2] & 0x3f]; + in += 3; + } + + // Handle remaining bytes + switch(len % 3) { + case 2: + *out++ = tab[(in[0] & 0xfc) >> 2]; + *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; + *out++ = tab[(in[1] & 0x0f) << 2]; + *out++ = '='; + break; + case 1: + *out++ = tab[(in[0] & 0xfc) >> 2]; + *out++ = tab[(in[0] & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + break; + } + + output.resize(out - &output[0]); + return output; + } + + inline static std::string decode(const std::string& input) { + std::string output; + output.resize(decoded_size(input.size())); + + const unsigned char* in = reinterpret_cast(input.data()); + char* out = &output[0]; + size_t len = input.size(); + + unsigned char c3[3], c4[4]; + int i = 0; + int j = 0; + + auto const inverse = get_inverse(); + + while(len-- && *in != '=') { + auto const v = inverse[*in]; + if(v == -1) + break; + ++in; + c4[i] = v; + if(++i == 4) { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for(i = 0; i < 3; i++) + *out++ = c3[i]; + i = 0; + } + } + + if(i) { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for(j = 0; j < i - 1; j++) + *out++ = c3[j]; + } + + output.resize(out - &output[0]); + return output; + } + + inline static void decode(unsigned char const * input, + const std::size_t len_in, + unsigned char * const output, + std::size_t& len_out) { + // Same as above, but with output passed as reference to avoid reallocation, more C-like + // this assumes that output is allocated and of the max correct size + // len_out is the size of the output buffer + + std::size_t len = len_in; + unsigned char* out = output; + unsigned char* in = const_cast(input); + + unsigned char c3[3], c4[4]; + int i = 0; + int j = 0; + + auto const inverse = get_inverse(); + + while(len-- && *in != '=') { + auto const v = inverse[*in]; + if(v == -1) + break; + ++in; + c4[i] = v; + if(++i == 4) { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for(i = 0; i < 3; i++) + *out++ = c3[i]; + i = 0; + } + } + + if(i) { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for(j = 0; j < i - 1; j++) + *out++ = c3[j]; + } + + len_out = out - output; + } + +}; + +#endif // BASE64_HPP diff --git a/cpp/src/KIM_SharedLibrary.cpp b/cpp/src/KIM_SharedLibrary.cpp index 32b09ff4..f8240041 100644 --- a/cpp/src/KIM_SharedLibrary.cpp +++ b/cpp/src/KIM_SharedLibrary.cpp @@ -37,6 +37,7 @@ #endif #include #include +#include #include // IWYU pragma: keep // For macOS #ifndef KIM_SHARED_LIBRARY_HPP_ @@ -55,6 +56,10 @@ #include "KIM_SharedLibrarySchema.hpp" #endif +#ifndef BASE64_HPP +#include "base64.hpp" // For base64 decoding +#endif + namespace { static void * const referencePointForKIM_Library = NULL; @@ -649,7 +654,28 @@ int SharedLibrary::WriteParameterFileDirectory() std::ofstream fl; fl.open(specificationFilePathName.string().c_str(), std::ifstream::out | std::ifstream::binary); - fl.write(reinterpret_cast(specificationData), len); + const std::size_t FILE_NEWLINE_PAD_OFFSET = 1; + const std::size_t PERLINE_PAD = 1; + + int usable_chars = len - FILE_NEWLINE_PAD_OFFSET*2; + const auto file_start_ptr = specificationData + FILE_NEWLINE_PAD_OFFSET; + + std::array binary_line; + + const int char_per_line = static_cast(Base64::MAX_BASE64_WIDTH); + + int char_remaining = usable_chars; + std::size_t out_len = 0; + unsigned int offset = 0; + + while(char_remaining > 0){ + int n_chars_this_line = std::min(char_per_line, char_remaining); + Base64::decode(file_start_ptr + offset, n_chars_this_line, binary_line.data(), out_len); + fl.write(reinterpret_cast(binary_line.data()), out_len); + char_remaining -= (char_per_line + PERLINE_PAD); + offset += (char_per_line + PERLINE_PAD); + } + if (!fl) { LOG_ERROR("Unable to get write parameter file."); @@ -678,9 +704,31 @@ int SharedLibrary::WriteParameterFileDirectory() FILESYSTEM::Path const parameterFilePathName = parameterFileDirectoryName_ / parameterFileName; std::ofstream fl; + fl.open(parameterFilePathName.string().c_str(), std::ifstream::out | std::ifstream::binary); - fl.write(reinterpret_cast(strPtr), length); + + const std::size_t FILE_NEWLINE_PAD_OFFSET = 1; + const std::size_t PERLINE_PAD = 1; + + int usable_chars = length - FILE_NEWLINE_PAD_OFFSET*2; + const auto file_start_ptr = strPtr + FILE_NEWLINE_PAD_OFFSET; + + std::array binary_line; + + const int char_per_line = static_cast(Base64::MAX_BASE64_WIDTH); + + int char_remaining = usable_chars; + std::size_t out_len = 0; + unsigned int offset = 0; + + while(char_remaining > 0){ + int n_chars_this_line = std::min(char_per_line, char_remaining); + Base64::decode(file_start_ptr + offset, n_chars_this_line, binary_line.data(), out_len); + fl.write(reinterpret_cast(binary_line.data()), out_len); + char_remaining -= (char_per_line + PERLINE_PAD); + offset += (char_per_line + PERLINE_PAD); + } if (!fl) { LOG_ERROR("Unable to get write parameter file."); diff --git a/utils/encode.cpp b/utils/encode.cpp new file mode 100644 index 00000000..19f3d19a --- /dev/null +++ b/utils/encode.cpp @@ -0,0 +1,71 @@ +#include "base64.hpp" +#include +#include +#include +#include +#include + +void writeEncodedFile(std::string & filename, std::string & output_filename){ + + std::string parsed_file_string = filename; + std::replace(parsed_file_string.begin(), parsed_file_string.end(), '.', '_'); + std::replace(parsed_file_string.begin(), parsed_file_string.end(), '-', '_'); + std::string encoded_file_name = output_filename; + + std::ifstream input_file(filename, std::ios::binary); + std::ofstream output_file(encoded_file_name); + + unsigned int len = 0; + + const size_t chunk = 1024 * 48; // 48 kb buffer, multiple of 3, base64 3bytes to 4 char + const size_t linewidth = 76; + std::string buffer; + buffer.reserve(chunk); + size_t linepos =0 ; + + std::string header = "extern unsigned char " + parsed_file_string + "[] = R\"(\n"; + // std::string header = "constexpr unsigned char " + parsed_file_string + "[] = R\"(\n"; + output_file.write(header.data(), header.length()); + + std::vector raw_buffer(chunk); + + while (input_file.read(raw_buffer.data(), chunk) || input_file.gcount()){ + buffer.assign(raw_buffer.data(), input_file.gcount()); // assign exact string from raw buffer + // If raw < chunk this will deal with it + std::string encoded = Base64::encode(buffer); + for (char &c : encoded){ + output_file.put(c); + linepos++; + len++; + if (linepos >= linewidth){ + output_file.put('\n'); + len++; + linepos = 0; + } + } + } + if (linepos > 0) output_file.put('\n'); + + len += 2; // two \n in beginning and end + std::string footer = ")\";\nextern unsigned int " + parsed_file_string + "_len = " + std::to_string(len) + ";\n"; + + output_file.write(footer.data(), footer.length()); + input_file.close(); + output_file.close(); +} + +int main(int argc, char * argv[]){ + if (argc != 3){ + for(int i = 0; i < argc; i++){ + std::cout << ">>>>>" << argv[i] << "\n"; + } + std::cerr << "Improper arguments, please provide just the encoding filename\n"; + return 1; + } + + std::string filename = argv[1]; + std::string output_filename = argv[2]; + writeEncodedFile(filename, output_filename); + + return 0; +} From c3ba3f46b06dec6bbf08150c6e0563a7b8de2708 Mon Sep 17 00:00:00 2001 From: nav-mohan Date: Tue, 7 Jan 2025 18:43:38 -0500 Subject: [PATCH 02/14] Build kim_base64_encode Specify build and installation instructions for kim_base64_encode. kim_base64_encode is a critical dependant for building model/model-driver shared-libraries. kim_base64_encode must be built before kim-api. kim_base64_encode will be installed in the CMAKE_INSTALL_RELOC_BINDIR for system-wide access during compile-time of model/model-driver shared-libraries. --- CMakeLists.txt | 11 ++++++++++- cmake/items-macros.cmake.in | 5 ++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 26b5a05e..14deefa1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug") set(_LOG_MAX "DEBUG") else() set(_LOG_MAX "INFORMATION") - set(CMAKE_BUILD_TYPE "Release") # Default to Release, soves KIM slowdown complaints + set(CMAKE_BUILD_TYPE "Release") # Default to Release, solves KIM slowdown complaints endif() # C++ Language Standard, enforced to be C++11 @@ -160,6 +160,12 @@ if(NOT WIN32 OR CYGWIN) target_link_libraries(kim-api ${CMAKE_DL_LIBS}) endif() +add_executable(kim-base64-encode "${PROJECT_SOURCE_DIR}/utils/encode.cpp") +target_include_directories(kim-base64-encode PRIVATE "${PROJECT_SOURCE_DIR}/cpp/include") +set_target_properties(kim-base64-encode PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/utils" ) +SET(XXD_EXECUTABLE ${PROJECT_BINARY_DIR}/utils/kim-base64-encode) +add_dependencies(kim-api kim-base64-encode) + # Add install rules for kim-api # install(TARGETS kim-api @@ -180,6 +186,9 @@ install( DESTINATION "${CMAKE_INSTALL_RELOC_DOCDIR}" ) +# install kim-base64-encode in CMAKE_INSTALL_RELOC_BINDIR +install(TARGETS kim-base64-encode + RUNTIME DESTINATION "${CMAKE_INSTALL_RELOC_BINDIR}") # Add include subdirectories # diff --git a/cmake/items-macros.cmake.in b/cmake/items-macros.cmake.in index b2701553..0419778d 100644 --- a/cmake/items-macros.cmake.in +++ b/cmake/items-macros.cmake.in @@ -503,10 +503,9 @@ endmacro() # function(_add_custom_command_blob_to_cpp _dirin _filein _fileout) include(FindPackageMessage) - # find_program(XXD_EXECUTABLE "xxd") - find_program(XXD_EXECUTABLE "kim_base64_encode") + find_program(XXD_EXECUTABLE "kim-base64-encode") if(XXD_EXECUTABLE) - find_package_message(xxd "Found kim_base64_encode: (${XXD_EXECUTABLE})" "found") + find_package_message(xxd "Found kim-base64-encode: (${XXD_EXECUTABLE})" "found") string(MAKE_C_IDENTIFIER ${_filein} _cfilein) set(_edit_xxd_output "${CMAKE_CURRENT_BINARY_DIR}/EditXXDOutput.cmake") From 59d010091a6febff5d7961eccca6493174b4192d Mon Sep 17 00:00:00 2001 From: Amit Gupta Date: Thu, 9 Jan 2025 09:55:07 -0600 Subject: [PATCH 03/14] Modified item-macros.cmake to remove the XXD fallback. It is incompatible now. Stylistic changes: base64.hpp moved to KIM_Base64.hpp KIM_Base64.hpp edited to have 2 space indent Function names/variable names to follow PascalCase or camelCase (like other KIM-API files) Doxygen style docstrings TODO: Licensing issue discussion with Ryan --- CMakeLists.txt | 9 +- cmake/items-macros.cmake.in | 52 ++++---- cpp/include/KIM_Base64.hpp | 218 ++++++++++++++++++++++++++++++++++ cpp/include/base64.hpp | 191 ----------------------------- cpp/src/KIM_SharedLibrary.cpp | 14 +-- utils/KIM_Base64Encode.cpp | 82 +++++++++++++ utils/encode.cpp | 71 ----------- 7 files changed, 342 insertions(+), 295 deletions(-) create mode 100644 cpp/include/KIM_Base64.hpp delete mode 100644 cpp/include/base64.hpp create mode 100644 utils/KIM_Base64Encode.cpp delete mode 100644 utils/encode.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 14deefa1..92f5ae46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,8 @@ else() set(CMAKE_BUILD_TYPE "Release") # Default to Release, solves KIM slowdown complaints endif() +message(STATUS "KIM-API Build Type: ${CMAKE_BUILD_TYPE}") + # C++ Language Standard, enforced to be C++11 set(CMAKE_CXX_STANDARD 11) @@ -160,10 +162,13 @@ if(NOT WIN32 OR CYGWIN) target_link_libraries(kim-api ${CMAKE_DL_LIBS}) endif() -add_executable(kim-base64-encode "${PROJECT_SOURCE_DIR}/utils/encode.cpp") +add_executable(kim-base64-encode "${PROJECT_SOURCE_DIR}/utils/KIM_Base64Encode.cpp") target_include_directories(kim-base64-encode PRIVATE "${PROJECT_SOURCE_DIR}/cpp/include") set_target_properties(kim-base64-encode PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/utils" ) -SET(XXD_EXECUTABLE ${PROJECT_BINARY_DIR}/utils/kim-base64-encode) + +# Keeping the name XXD for now, as otherwise this will result in much larger refactoring +# But everywhere, XXD = kim-base64-encode +set(XXD_EXECUTABLE ${PROJECT_BINARY_DIR}/utils/kim-base64-encode) add_dependencies(kim-api kim-base64-encode) # Add install rules for kim-api diff --git a/cmake/items-macros.cmake.in b/cmake/items-macros.cmake.in index 0419778d..3ccd08db 100644 --- a/cmake/items-macros.cmake.in +++ b/cmake/items-macros.cmake.in @@ -530,30 +530,34 @@ function(_add_custom_command_blob_to_cpp _dirin _filein _fileout) unset(_cfilein) unset(_edit_xxd_output) else() - find_package_message(xxd "Missing xxd: Falling back to less efficient cmake implementation." "missing") - - string(MAKE_C_IDENTIFIER ${_filein} _cfilein) - set(_blob_to_source "${CMAKE_CURRENT_BINARY_DIR}/BlobToCSource.cmake") - if(NOT EXISTS "${_blob_to_source}") - file(WRITE "${_blob_to_source}" # use a bracket argument to avoid ugly escaping - [=[ - # This file was automatically generated by CMake; do not edit. - file(READ "${filein}" _content HEX) - string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," _content "${_content}") - string(REGEX REPLACE ",$" "" _content "${_content}") - file(WRITE "${fileout}" "unsigned char const ${varname}[] = { ${_content} };\n") - file(APPEND "${fileout}" "unsigned int const ${varname}_len = sizeof( ${varname} );\n") - unset(_content) - ]=] - ) - endif() - add_custom_command(OUTPUT ${_fileout} - COMMAND ${CMAKE_COMMAND} "-Dvarname=${_cfilein}" "-Dfilein=${_filein}" "-Dfileout=${_fileout}" -P "${_blob_to_source}" - DEPENDS "${_dirin}/${_filein}" "${_blob_to_source}" - WORKING_DIRECTORY "${_dirin}" - ) - unset(_cfilein) - unset(_blob_to_source) + # No backup yet + # Possible Solutions: 1) CMAKE only implementation of base64 encoding? + # 2) use system base64/openssl binaries? + message(FATAL_ERROR "Missing kim-base64-encode: Please check the KIM-API installation.") + # find_package_message(xxd "Missing xxd: Falling back to less efficient cmake implementation." "missing") + + # string(MAKE_C_IDENTIFIER ${_filein} _cfilein) + # set(_blob_to_source "${CMAKE_CURRENT_BINARY_DIR}/BlobToCSource.cmake") + # if(NOT EXISTS "${_blob_to_source}") + # file(WRITE "${_blob_to_source}" # use a bracket argument to avoid ugly escaping + # [=[ + # # This file was automatically generated by CMake; do not edit. + # file(READ "${filein}" _content HEX) + # string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," _content "${_content}") + # string(REGEX REPLACE ",$" "" _content "${_content}") + # file(WRITE "${fileout}" "unsigned char const ${varname}[] = { ${_content} };\n") + # file(APPEND "${fileout}" "unsigned int const ${varname}_len = sizeof( ${varname} );\n") + # unset(_content) + # ]=] + # ) + # endif() + # add_custom_command(OUTPUT ${_fileout} + # COMMAND ${CMAKE_COMMAND} "-Dvarname=${_cfilein}" "-Dfilein=${_filein}" "-Dfileout=${_fileout}" -P "${_blob_to_source}" + # DEPENDS "${_dirin}/${_filein}" "${_blob_to_source}" + # WORKING_DIRECTORY "${_dirin}" + # ) + # unset(_cfilein) + # unset(_blob_to_source) endif() endfunction(_add_custom_command_blob_to_cpp) diff --git a/cpp/include/KIM_Base64.hpp b/cpp/include/KIM_Base64.hpp new file mode 100644 index 00000000..7783907c --- /dev/null +++ b/cpp/include/KIM_Base64.hpp @@ -0,0 +1,218 @@ +// ASK RYAN: +// License for this file? The original class was part of Boost, and distributed under +// the Boost Software License. It allows for inclusion of the code in other projects +// so no issues there, but Boost Licence explicitly asks that the Boost License +// must be included along with the LGPL-2.1-or-later license. Is it sufficient to +// include the Boost License in the header file as well, or will it need to be included +// in base project as well? +// Rewrite the class? Though it will be similar to current version with minor changes + +// Currently distributed under Boost Software License + +#ifndef BASE64_HPP +#define BASE64_HPP + +#include +#include +#include + +class Base64 { + // Based on boost https://www.boost.org/doc/libs/1_66_0/boost/beast/core/detail/base64.hpp + private: + /// \brief Base64 alphabet table + inline static const char* get_alphabet() { + static const char tab[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + return tab; + } + + /// \brief Base64 inverse lookup table + inline static const signed char* get_inverse() { + static const signed char tab[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 + }; + return tab; + } + + + public: + Base64() = delete; + + /// \brief Calculate encoded size + /// + /// \param n The size of the input + /// \return The size of the encoded output + inline static constexpr size_t encoded_size(size_t n) { + return 4 * ((n + 2) / 3); + } + + /// \brief Calculate maximum decoded size + /// + /// \param n The size of the input + /// \return The maximum size of the decoded output + inline static constexpr size_t decoded_size(size_t n) { + return n / 4 * 3; + } + + // width of encoded output + constexpr static std::size_t MAX_BASE64_WIDTH = 76; + constexpr static std::size_t MAX_BINARY_WIDTH = MAX_BASE64_WIDTH/4 * 3; + + + /// \brief Encode a string + /// + /// \param input The input string + /// \return The encoded string + inline static std::string encode(const std::string& input) { + std::string output; + output.resize(encoded_size(input.size())); + + const char* in = input.data(); + char* out = &output[0]; + size_t len = input.size(); + auto const tab = get_alphabet(); + + // Process 3-byte chunks + for(size_t n = len / 3; n--;) { + *out++ = tab[(in[0] & 0xfc) >> 2]; + *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; + *out++ = tab[((in[2] & 0xc0) >> 6) + ((in[1] & 0x0f) << 2)]; + *out++ = tab[in[2] & 0x3f]; + in += 3; + } + + // Handle remaining bytes + switch(len % 3) { + case 2: + *out++ = tab[(in[0] & 0xfc) >> 2]; + *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; + *out++ = tab[(in[1] & 0x0f) << 2]; + *out++ = '='; + break; + case 1: + *out++ = tab[(in[0] & 0xfc) >> 2]; + *out++ = tab[(in[0] & 0x03) << 4]; + *out++ = '='; + *out++ = '='; + break; + } + + output.resize(out - &output[0]); + return output; + } + + /// \brief Encode a string, counterpart to encode + /// + /// \param input The input string + /// \return decoded string + inline static std::string decode(const std::string& input) { + std::string output; + output.resize(decoded_size(input.size())); + + const unsigned char* in = reinterpret_cast(input.data()); + char* out = &output[0]; + size_t len = input.size(); + + unsigned char c3[3], c4[4]; + int i = 0; + int j = 0; + + auto const inverse = get_inverse(); + + while(len-- && *in != '=') { + auto const v = inverse[*in]; + if(v == -1) + break; + ++in; + c4[i] = v; + if(++i == 4) { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for(i = 0; i < 3; i++) + *out++ = c3[i]; + i = 0; + } + } + + if(i) { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for(j = 0; j < i - 1; j++) + *out++ = c3[j]; + } + + output.resize(out - &output[0]); + return output; + } + + /// \brief Decode a string, but with a more C-like interface + /// + /// \param input pointer to input buffer + /// \param len_in length of the input buffer + /// \param output pointer to store the output buffer + /// \param len_out length of the output buffer written + inline static void decode(unsigned char const * input, + const std::size_t len_in, + unsigned char * const output, + std::size_t& len_out) { + std::size_t len = len_in; + unsigned char* out = output; + auto in = const_cast(input); + + unsigned char c3[3], c4[4]; + int i = 0; + int j = 0; + + auto const inverse = get_inverse(); + + while(len-- && *in != '=') { + auto const v = inverse[*in]; + if(v == -1) + break; + ++in; + c4[i] = v; + if(++i == 4) { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for(i = 0; i < 3; i++) + *out++ = c3[i]; + i = 0; + } + } + + if(i) { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for(j = 0; j < i - 1; j++) + *out++ = c3[j]; + } + + len_out = out - output; + } + +}; + +#endif // BASE64_HPP diff --git a/cpp/include/base64.hpp b/cpp/include/base64.hpp deleted file mode 100644 index e2cc91f1..00000000 --- a/cpp/include/base64.hpp +++ /dev/null @@ -1,191 +0,0 @@ -#ifndef BASE64_HPP -#define BASE64_HPP - -#include -#include -#include - -class Base64 { -// Based on boost https://www.boost.org/doc/libs/1_66_0/boost/beast/core/detail/base64.hpp -private: - // Base64 alphabet table - inline static const char* get_alphabet() { - static const char tab[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - return tab; - } - - // Base64 inverse lookup table - inline static const signed char* get_inverse() { - static const signed char tab[] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, - -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, - -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 - }; - return tab; - } - - -public: - // Prevent instantiation - Base64() = delete; - // Calculate encoded size - inline static constexpr size_t encoded_size(size_t n) { - return 4 * ((n + 2) / 3); - } - - // Calculate decoded size - inline static constexpr size_t decoded_size(size_t n) { - return n / 4 * 3; - } - - constexpr static std::size_t MAX_BASE64_WIDTH = 76; - constexpr static std::size_t MAX_BINARY_WIDTH = MAX_BASE64_WIDTH/4 * 3; - - - - inline static std::string encode(const std::string& input) { - std::string output; - output.resize(encoded_size(input.size())); - - const char* in = input.data(); - char* out = &output[0]; - size_t len = input.size(); - auto const tab = get_alphabet(); - - // Process 3-byte chunks - for(size_t n = len / 3; n--;) { - *out++ = tab[(in[0] & 0xfc) >> 2]; - *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; - *out++ = tab[((in[2] & 0xc0) >> 6) + ((in[1] & 0x0f) << 2)]; - *out++ = tab[in[2] & 0x3f]; - in += 3; - } - - // Handle remaining bytes - switch(len % 3) { - case 2: - *out++ = tab[(in[0] & 0xfc) >> 2]; - *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; - *out++ = tab[(in[1] & 0x0f) << 2]; - *out++ = '='; - break; - case 1: - *out++ = tab[(in[0] & 0xfc) >> 2]; - *out++ = tab[(in[0] & 0x03) << 4]; - *out++ = '='; - *out++ = '='; - break; - } - - output.resize(out - &output[0]); - return output; - } - - inline static std::string decode(const std::string& input) { - std::string output; - output.resize(decoded_size(input.size())); - - const unsigned char* in = reinterpret_cast(input.data()); - char* out = &output[0]; - size_t len = input.size(); - - unsigned char c3[3], c4[4]; - int i = 0; - int j = 0; - - auto const inverse = get_inverse(); - - while(len-- && *in != '=') { - auto const v = inverse[*in]; - if(v == -1) - break; - ++in; - c4[i] = v; - if(++i == 4) { - c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); - c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); - c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; - - for(i = 0; i < 3; i++) - *out++ = c3[i]; - i = 0; - } - } - - if(i) { - c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); - c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); - c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; - - for(j = 0; j < i - 1; j++) - *out++ = c3[j]; - } - - output.resize(out - &output[0]); - return output; - } - - inline static void decode(unsigned char const * input, - const std::size_t len_in, - unsigned char * const output, - std::size_t& len_out) { - // Same as above, but with output passed as reference to avoid reallocation, more C-like - // this assumes that output is allocated and of the max correct size - // len_out is the size of the output buffer - - std::size_t len = len_in; - unsigned char* out = output; - unsigned char* in = const_cast(input); - - unsigned char c3[3], c4[4]; - int i = 0; - int j = 0; - - auto const inverse = get_inverse(); - - while(len-- && *in != '=') { - auto const v = inverse[*in]; - if(v == -1) - break; - ++in; - c4[i] = v; - if(++i == 4) { - c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); - c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); - c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; - - for(i = 0; i < 3; i++) - *out++ = c3[i]; - i = 0; - } - } - - if(i) { - c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); - c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); - c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; - - for(j = 0; j < i - 1; j++) - *out++ = c3[j]; - } - - len_out = out - output; - } - -}; - -#endif // BASE64_HPP diff --git a/cpp/src/KIM_SharedLibrary.cpp b/cpp/src/KIM_SharedLibrary.cpp index f8240041..777d6e96 100644 --- a/cpp/src/KIM_SharedLibrary.cpp +++ b/cpp/src/KIM_SharedLibrary.cpp @@ -57,7 +57,7 @@ #endif #ifndef BASE64_HPP -#include "base64.hpp" // For base64 decoding +#include "KIM_Base64.hpp" // For base64 decoding #endif namespace @@ -657,10 +657,10 @@ int SharedLibrary::WriteParameterFileDirectory() const std::size_t FILE_NEWLINE_PAD_OFFSET = 1; const std::size_t PERLINE_PAD = 1; - int usable_chars = len - FILE_NEWLINE_PAD_OFFSET*2; + int usable_chars = static_cast(len) - FILE_NEWLINE_PAD_OFFSET*2; const auto file_start_ptr = specificationData + FILE_NEWLINE_PAD_OFFSET; - std::array binary_line; + std::array binary_line{}; const int char_per_line = static_cast(Base64::MAX_BASE64_WIDTH); @@ -671,7 +671,7 @@ int SharedLibrary::WriteParameterFileDirectory() while(char_remaining > 0){ int n_chars_this_line = std::min(char_per_line, char_remaining); Base64::decode(file_start_ptr + offset, n_chars_this_line, binary_line.data(), out_len); - fl.write(reinterpret_cast(binary_line.data()), out_len); + fl.write(reinterpret_cast(binary_line.data()), static_cast(out_len)); char_remaining -= (char_per_line + PERLINE_PAD); offset += (char_per_line + PERLINE_PAD); } @@ -711,10 +711,10 @@ int SharedLibrary::WriteParameterFileDirectory() const std::size_t FILE_NEWLINE_PAD_OFFSET = 1; const std::size_t PERLINE_PAD = 1; - int usable_chars = length - FILE_NEWLINE_PAD_OFFSET*2; + int usable_chars = static_cast(length) - FILE_NEWLINE_PAD_OFFSET*2; const auto file_start_ptr = strPtr + FILE_NEWLINE_PAD_OFFSET; - std::array binary_line; + std::array binary_line{}; const int char_per_line = static_cast(Base64::MAX_BASE64_WIDTH); @@ -725,7 +725,7 @@ int SharedLibrary::WriteParameterFileDirectory() while(char_remaining > 0){ int n_chars_this_line = std::min(char_per_line, char_remaining); Base64::decode(file_start_ptr + offset, n_chars_this_line, binary_line.data(), out_len); - fl.write(reinterpret_cast(binary_line.data()), out_len); + fl.write(reinterpret_cast(binary_line.data()), static_cast(out_len)); char_remaining -= (char_per_line + PERLINE_PAD); offset += (char_per_line + PERLINE_PAD); } diff --git a/utils/KIM_Base64Encode.cpp b/utils/KIM_Base64Encode.cpp new file mode 100644 index 00000000..d9fba03d --- /dev/null +++ b/utils/KIM_Base64Encode.cpp @@ -0,0 +1,82 @@ +// +// Distributed under the existing KIM-API License. +// This is a simple program that reads a file and encodes it in base64 +// and writes it to a new file, like XXD but with base64 encoding + +#include "KIM_Base64.hpp" +#include +#include +#include +#include +#include + +/// \brief Write the Base^4 encode binary file as a C++ source file, emulating XXD +/// +/// \param[in] fileName The name of the file to encode +/// \param[in] outputFileName The name of the file to write the encoded file to +/// \sa Base64::encode +void WriteEncodedFile(std::string & fileName, std::string & outputFileName){ + + std::string xxdFormatFileName = fileName; + // Make the file names compatible with xxd variable name format + std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '.', '_'); + std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '-', '_'); + + std::ifstream inputFile(fileName, std::ios::binary); + std::ofstream outputFile(outputFileName); + + unsigned int len = 0; + + const size_t chunk = 1024 * 48; // 48 kb buffer, multiple of 3, + // base64 writes 3 bytes to 4 char + std::string buffer; // buffer to store known amount of raw data + buffer.reserve(chunk); + size_t linepos =0 ; + + // C++ const is different from C const (equivalent to static const), + // so we need to use extern + std::string header = "extern unsigned char " + xxdFormatFileName + "[] = R\"(\n"; + + outputFile.write(header.data(), header.length()); + + std::vector rawBuffer(chunk); // buffer to store raw io data + + while (inputFile.read(rawBuffer.data(), chunk) || inputFile.gcount()){ + buffer.assign(rawBuffer.data(), inputFile.gcount()); // assign exact string from raw buffer + // If raw < chunk this will deal with it + + // Possible optimization: stored encoded data in buffer pointer, like the decode function + std::string encoded = Base64::encode(buffer); + for (char &c : encoded){ + outputFile.put(c); + linepos++; + len++; + if (linepos >= Base64::MAX_BASE64_WIDTH){ + outputFile.put('\n'); + len++; + linepos = 0; + } + } + } + if (linepos > 0) outputFile.put('\n'); + + len += 2; // two \n in beginning and end + std::string footer = ")\";\nextern unsigned int " + xxdFormatFileName + "_len = " + std::to_string(len) + ";\n"; + + outputFile.write(footer.data(), static_cast(footer.length())); + inputFile.close(); + outputFile.close(); +} + +int main(int argc, char * argv[]){ + if (argc != 3){ + std::cerr << "Improper arguments, please provide input and output file name\n"; + return 1; + } + + std::string fileName = argv[1]; + std::string outputFileName = argv[2]; + WriteEncodedFile(fileName, outputFileName); + + return 0; +} diff --git a/utils/encode.cpp b/utils/encode.cpp deleted file mode 100644 index 19f3d19a..00000000 --- a/utils/encode.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include "base64.hpp" -#include -#include -#include -#include -#include - -void writeEncodedFile(std::string & filename, std::string & output_filename){ - - std::string parsed_file_string = filename; - std::replace(parsed_file_string.begin(), parsed_file_string.end(), '.', '_'); - std::replace(parsed_file_string.begin(), parsed_file_string.end(), '-', '_'); - std::string encoded_file_name = output_filename; - - std::ifstream input_file(filename, std::ios::binary); - std::ofstream output_file(encoded_file_name); - - unsigned int len = 0; - - const size_t chunk = 1024 * 48; // 48 kb buffer, multiple of 3, base64 3bytes to 4 char - const size_t linewidth = 76; - std::string buffer; - buffer.reserve(chunk); - size_t linepos =0 ; - - std::string header = "extern unsigned char " + parsed_file_string + "[] = R\"(\n"; - // std::string header = "constexpr unsigned char " + parsed_file_string + "[] = R\"(\n"; - output_file.write(header.data(), header.length()); - - std::vector raw_buffer(chunk); - - while (input_file.read(raw_buffer.data(), chunk) || input_file.gcount()){ - buffer.assign(raw_buffer.data(), input_file.gcount()); // assign exact string from raw buffer - // If raw < chunk this will deal with it - std::string encoded = Base64::encode(buffer); - for (char &c : encoded){ - output_file.put(c); - linepos++; - len++; - if (linepos >= linewidth){ - output_file.put('\n'); - len++; - linepos = 0; - } - } - } - if (linepos > 0) output_file.put('\n'); - - len += 2; // two \n in beginning and end - std::string footer = ")\";\nextern unsigned int " + parsed_file_string + "_len = " + std::to_string(len) + ";\n"; - - output_file.write(footer.data(), footer.length()); - input_file.close(); - output_file.close(); -} - -int main(int argc, char * argv[]){ - if (argc != 3){ - for(int i = 0; i < argc; i++){ - std::cout << ">>>>>" << argv[i] << "\n"; - } - std::cerr << "Improper arguments, please provide just the encoding filename\n"; - return 1; - } - - std::string filename = argv[1]; - std::string output_filename = argv[2]; - writeEncodedFile(filename, output_filename); - - return 0; -} From 0596fa3f4e46f4c83d1d1248bab5cfd48933f567 Mon Sep 17 00:00:00 2001 From: nav-mohan Date: Sun, 12 Jan 2025 15:00:36 -0500 Subject: [PATCH 04/14] Relocate KIM_Base64Encode from utils/ to cpp/src/ --- CMakeLists.txt | 2 +- {utils => cpp/src}/KIM_Base64Encode.cpp | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {utils => cpp/src}/KIM_Base64Encode.cpp (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 92f5ae46..6a2fda44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,7 +162,7 @@ if(NOT WIN32 OR CYGWIN) target_link_libraries(kim-api ${CMAKE_DL_LIBS}) endif() -add_executable(kim-base64-encode "${PROJECT_SOURCE_DIR}/utils/KIM_Base64Encode.cpp") +add_executable(kim-base64-encode "${PROJECT_SOURCE_DIR}/cpp/src/KIM_Base64Encode.cpp") target_include_directories(kim-base64-encode PRIVATE "${PROJECT_SOURCE_DIR}/cpp/include") set_target_properties(kim-base64-encode PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/utils" ) diff --git a/utils/KIM_Base64Encode.cpp b/cpp/src/KIM_Base64Encode.cpp similarity index 100% rename from utils/KIM_Base64Encode.cpp rename to cpp/src/KIM_Base64Encode.cpp From 8c01bb39c762b4c6190b4d75344f58f0405a4102 Mon Sep 17 00:00:00 2001 From: nav-mohan Date: Sun, 12 Jan 2025 17:11:12 -0500 Subject: [PATCH 05/14] Work In Progress - Port code to C++98 This is a work in progress and the code is not expected to compile at this stage. A number of small changes have been made to port the code from c++11 to c++98. A pending change is to replace the string-literals with escaped characters. --- CMakeLists.txt | 2 +- cpp/include/KIM_Base64.hpp | 36 +++++++++++------------ cpp/src/KIM_Base64Encode.cpp | 54 +++++++++++++++++++++++++---------- cpp/src/KIM_SharedLibrary.cpp | 11 +++---- 4 files changed, 63 insertions(+), 40 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a2fda44..8c2ac3b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,7 +51,7 @@ endif() message(STATUS "KIM-API Build Type: ${CMAKE_BUILD_TYPE}") # C++ Language Standard, enforced to be C++11 -set(CMAKE_CXX_STANDARD 11) +# set(CMAKE_CXX_STANDARD 11) set_cache_with_fallback(KIM_API_LOG_MAXIMUM_LEVEL "${_LOG_MAX}" STRING "Maximum log verbosity") unset(_LOG_MAX) diff --git a/cpp/include/KIM_Base64.hpp b/cpp/include/KIM_Base64.hpp index 7783907c..8ae84322 100644 --- a/cpp/include/KIM_Base64.hpp +++ b/cpp/include/KIM_Base64.hpp @@ -14,13 +14,14 @@ #include #include -#include +// #include -class Base64 { +namespace Base64 { // Based on boost https://www.boost.org/doc/libs/1_66_0/boost/beast/core/detail/base64.hpp - private: + namespace + { /// \brief Base64 alphabet table - inline static const char* get_alphabet() { + inline const char* get_alphabet() { static const char tab[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; return tab; @@ -48,16 +49,13 @@ class Base64 { }; return tab; } - - - public: - Base64() = delete; + } // anonymous namespace /// \brief Calculate encoded size /// /// \param n The size of the input /// \return The size of the encoded output - inline static constexpr size_t encoded_size(size_t n) { + inline static size_t encoded_size(size_t n) { return 4 * ((n + 2) / 3); } @@ -65,13 +63,13 @@ class Base64 { /// /// \param n The size of the input /// \return The maximum size of the decoded output - inline static constexpr size_t decoded_size(size_t n) { + inline static size_t decoded_size(size_t n) { return n / 4 * 3; } // width of encoded output - constexpr static std::size_t MAX_BASE64_WIDTH = 76; - constexpr static std::size_t MAX_BINARY_WIDTH = MAX_BASE64_WIDTH/4 * 3; + const static std::size_t MAX_BASE64_WIDTH = 76; + const static std::size_t MAX_BINARY_WIDTH = MAX_BASE64_WIDTH/4 * 3; /// \brief Encode a string @@ -85,7 +83,7 @@ class Base64 { const char* in = input.data(); char* out = &output[0]; size_t len = input.size(); - auto const tab = get_alphabet(); + const char * tab = get_alphabet(); // Process 3-byte chunks for(size_t n = len / 3; n--;) { @@ -132,10 +130,10 @@ class Base64 { int i = 0; int j = 0; - auto const inverse = get_inverse(); + const signed char * inverse = get_inverse(); while(len-- && *in != '=') { - auto const v = inverse[*in]; + const signed char v = inverse[*in]; if(v == -1) break; ++in; @@ -176,16 +174,16 @@ class Base64 { std::size_t& len_out) { std::size_t len = len_in; unsigned char* out = output; - auto in = const_cast(input); + unsigned char * in = const_cast(input); unsigned char c3[3], c4[4]; int i = 0; int j = 0; - auto const inverse = get_inverse(); + const signed char * inverse = get_inverse(); while(len-- && *in != '=') { - auto const v = inverse[*in]; + const signed char v = inverse[*in]; if(v == -1) break; ++in; @@ -213,6 +211,6 @@ class Base64 { len_out = out - output; } -}; +} #endif // BASE64_HPP diff --git a/cpp/src/KIM_Base64Encode.cpp b/cpp/src/KIM_Base64Encode.cpp index d9fba03d..534cad70 100644 --- a/cpp/src/KIM_Base64Encode.cpp +++ b/cpp/src/KIM_Base64Encode.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include /// \brief Write the Base^4 encode binary file as a C++ source file, emulating XXD /// @@ -22,8 +24,14 @@ void WriteEncodedFile(std::string & fileName, std::string & outputFileName){ std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '.', '_'); std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '-', '_'); - std::ifstream inputFile(fileName, std::ios::binary); - std::ofstream outputFile(outputFileName); + // std::ifstream inputFile(fileName, std::ios::binary); + // std::ofstream outputFile(outputFileName); + + FILE *inputFile = fopen(fileName.c_str(), "rb"); + if(!inputFile) { std::cerr << "Error opening input file" << std::endl; return;} + + FILE *outputFile = fopen(outputFileName.c_str(), "w"); + if(!outputFile) {std::cerr << "Error opening output file" << std::endl; return;} unsigned int len = 0; @@ -35,37 +43,53 @@ void WriteEncodedFile(std::string & fileName, std::string & outputFileName){ // C++ const is different from C const (equivalent to static const), // so we need to use extern - std::string header = "extern unsigned char " + xxdFormatFileName + "[] = R\"(\n"; + // std::string header = "extern unsigned char " + xxdFormatFileName + "[] = R\"(\n"; + std::string header = "extern unsigned char " + xxdFormatFileName + "[] = \"("; - outputFile.write(header.data(), header.length()); + // outputFile.write(header.data(), header.length()); + fwrite(header.data(), sizeof(char), header.size(), outputFile); std::vector rawBuffer(chunk); // buffer to store raw io data - while (inputFile.read(rawBuffer.data(), chunk) || inputFile.gcount()){ - buffer.assign(rawBuffer.data(), inputFile.gcount()); // assign exact string from raw buffer + // while (inputFile.read(rawBuffer.data(), chunk) || inputFile.gcount()){ + while( size_t bytesRead = fread(rawBuffer.data(), 1, chunk, inputFile) ) + { + buffer.assign(rawBuffer.data(), bytesRead); // assign exact string from raw buffer // If raw < chunk this will deal with it // Possible optimization: stored encoded data in buffer pointer, like the decode function std::string encoded = Base64::encode(buffer); - for (char &c : encoded){ - outputFile.put(c); + for(size_t i = 0; i < encoded.size(); i++){ + char & c = encoded[i]; + // outputFile.put(c); + fputc(c,outputFile); linepos++; len++; if (linepos >= Base64::MAX_BASE64_WIDTH){ - outputFile.put('\n'); + // outputFile.put('\n'); + // fputc('\n',outputFile); + // fwrite("\\n",3,1,outputFile); len++; linepos = 0; } } } - if (linepos > 0) outputFile.put('\n'); + // if (linepos > 0) outputFile.put('\n'); + // if (linepos > 0) fputc('\n',outputFile); + // if (linepos > 0) fwrite("\\n",3,1,outputFile); - len += 2; // two \n in beginning and end - std::string footer = ")\";\nextern unsigned int " + xxdFormatFileName + "_len = " + std::to_string(len) + ";\n"; - outputFile.write(footer.data(), static_cast(footer.length())); - inputFile.close(); - outputFile.close(); + len += 2; // two \n in beginning and end + // std::string footer = ")\";\nextern unsigned int " + xxdFormatFileName + "_len = " + std::to_string(len) + ";\n"; + std::stringstream footerStream; + footerStream << ")\";\nextern unsigned int " << xxdFormatFileName << "_len = " << len << ";\n"; + std::string footer = footerStream.str(); + // outputFile.write(footer.data(), static_cast(footer.length())); + fwrite(footer.data(), sizeof(char), footer.size(), outputFile); + // inputFile.close(); + fclose(inputFile); + // outputFile.close(); + fclose(outputFile); } int main(int argc, char * argv[]){ diff --git a/cpp/src/KIM_SharedLibrary.cpp b/cpp/src/KIM_SharedLibrary.cpp index 777d6e96..839a1cbb 100644 --- a/cpp/src/KIM_SharedLibrary.cpp +++ b/cpp/src/KIM_SharedLibrary.cpp @@ -37,7 +37,6 @@ #endif #include #include -#include #include // IWYU pragma: keep // For macOS #ifndef KIM_SHARED_LIBRARY_HPP_ @@ -658,9 +657,10 @@ int SharedLibrary::WriteParameterFileDirectory() const std::size_t PERLINE_PAD = 1; int usable_chars = static_cast(len) - FILE_NEWLINE_PAD_OFFSET*2; - const auto file_start_ptr = specificationData + FILE_NEWLINE_PAD_OFFSET; + const unsigned char * file_start_ptr = specificationData + FILE_NEWLINE_PAD_OFFSET; - std::array binary_line{}; + // std::array binary_line{}; + std::vector binary_line(Base64::MAX_BINARY_WIDTH); const int char_per_line = static_cast(Base64::MAX_BASE64_WIDTH); @@ -712,9 +712,10 @@ int SharedLibrary::WriteParameterFileDirectory() const std::size_t PERLINE_PAD = 1; int usable_chars = static_cast(length) - FILE_NEWLINE_PAD_OFFSET*2; - const auto file_start_ptr = strPtr + FILE_NEWLINE_PAD_OFFSET; + const unsigned char * file_start_ptr = strPtr + FILE_NEWLINE_PAD_OFFSET; - std::array binary_line{}; + // std::array binary_line{}; + std::vector binary_line(Base64::MAX_BINARY_WIDTH); const int char_per_line = static_cast(Base64::MAX_BASE64_WIDTH); From 6f0876d7fd971a497544b6745f47545f3870df69 Mon Sep 17 00:00:00 2001 From: nav-mohan Date: Thu, 16 Jan 2025 16:27:46 -0500 Subject: [PATCH 06/14] CLI for kim-base64-encode Create the command-line-interface for kim-base64-encode following the docopt.org format keeping inline with other utils/kim-api-* binary executables --- CMakeLists.txt | 2 +- cpp/src/KIM_Base64Encode.cpp | 27 ++++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c2ac3b0..ebb95382 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,7 +163,7 @@ if(NOT WIN32 OR CYGWIN) endif() add_executable(kim-base64-encode "${PROJECT_SOURCE_DIR}/cpp/src/KIM_Base64Encode.cpp") -target_include_directories(kim-base64-encode PRIVATE "${PROJECT_SOURCE_DIR}/cpp/include") +target_include_directories(kim-base64-encode PRIVATE "${PROJECT_SOURCE_DIR}/cpp/include" "${CMAKE_CURRENT_BINARY_DIR}/cpp/include") set_target_properties(kim-base64-encode PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/utils" ) # Keeping the name XXD for now, as otherwise this will result in much larger refactoring diff --git a/cpp/src/KIM_Base64Encode.cpp b/cpp/src/KIM_Base64Encode.cpp index 534cad70..e0e903ca 100644 --- a/cpp/src/KIM_Base64Encode.cpp +++ b/cpp/src/KIM_Base64Encode.cpp @@ -4,6 +4,7 @@ // and writes it to a new file, like XXD but with base64 encoding #include "KIM_Base64.hpp" +#include "KIM_Version.hpp" #include #include #include @@ -92,12 +93,32 @@ void WriteEncodedFile(std::string & fileName, std::string & outputFileName){ fclose(outputFile); } +void usage(std::string name) +{ + size_t beg = name.find_last_of("/\\"); + if (beg != std::string::npos) name = name.substr(beg + 1, std::string::npos); + + // Follows docopt.org format + std::cerr << "Usage:\n" + << " " << name << " " << " " << "\n" + << " " << name << " " << "--version\n"; + // note: this interface is likely to change in future kim-api releases +} + int main(int argc, char * argv[]){ - if (argc != 3){ - std::cerr << "Improper arguments, please provide input and output file name\n"; - return 1; + + if ((argc == 2) && (std::string(argv[1]) == "--version")) + { + std::cout << KIM_VERSION_STRING << std::endl; + return 0; + } + if ((argc != 3)) + { + usage(argv[0]); + return -1; } + std::string fileName = argv[1]; std::string outputFileName = argv[2]; WriteEncodedFile(fileName, outputFileName); From cd8aeeb4cd33a426bd556b265dd160fba9584a55 Mon Sep 17 00:00:00 2001 From: Amit Gupta Date: Sat, 18 Jan 2025 18:43:33 -0600 Subject: [PATCH 07/14] Complete commit for C++98 compatible KIM-API large file --- CMakeLists.txt | 16 +- cpp/include/KIM_Base64.hpp | 216 ------------------------ cpp/src/KIM_Base64Encode.cpp | 127 -------------- cpp/src/KIM_SharedLibrary.cpp | 76 +++------ cpp/src/base64-encode/CMakeLists.txt | 18 ++ cpp/src/base64-encode/LICENSE | 23 +++ cpp/src/base64-encode/README | 30 ++++ cpp/src/base64-encode/base64-encode.cpp | 134 +++++++++++++++ cpp/src/base64-encode/base64.hpp | 188 +++++++++++++++++++++ 9 files changed, 426 insertions(+), 402 deletions(-) delete mode 100644 cpp/include/KIM_Base64.hpp delete mode 100644 cpp/src/KIM_Base64Encode.cpp create mode 100644 cpp/src/base64-encode/CMakeLists.txt create mode 100644 cpp/src/base64-encode/LICENSE create mode 100644 cpp/src/base64-encode/README create mode 100644 cpp/src/base64-encode/base64-encode.cpp create mode 100644 cpp/src/base64-encode/base64.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ebb95382..5a86ec1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,14 +162,14 @@ if(NOT WIN32 OR CYGWIN) target_link_libraries(kim-api ${CMAKE_DL_LIBS}) endif() -add_executable(kim-base64-encode "${PROJECT_SOURCE_DIR}/cpp/src/KIM_Base64Encode.cpp") -target_include_directories(kim-base64-encode PRIVATE "${PROJECT_SOURCE_DIR}/cpp/include" "${CMAKE_CURRENT_BINARY_DIR}/cpp/include") -set_target_properties(kim-base64-encode PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}/utils" ) # Keeping the name XXD for now, as otherwise this will result in much larger refactoring -# But everywhere, XXD = kim-base64-encode -set(XXD_EXECUTABLE ${PROJECT_BINARY_DIR}/utils/kim-base64-encode) -add_dependencies(kim-api kim-base64-encode) +# But everywhere, XXD = base64-encode +set(XXD_EXECUTABLE ${PROJECT_BINARY_DIR}/utils/base64-encode) +add_subdirectory(cpp/src/base64-encode ${CMAKE_BINARY_DIR}/utils/base64-encode) +add_dependencies(kim-api base64-encode) +install(TARGETS base64-encode + RUNTIME DESTINATION ${CMAKE_INSTALL_RELOC_BINDIR}) # Add install rules for kim-api # @@ -192,8 +192,8 @@ install( "${CMAKE_INSTALL_RELOC_DOCDIR}" ) # install kim-base64-encode in CMAKE_INSTALL_RELOC_BINDIR -install(TARGETS kim-base64-encode - RUNTIME DESTINATION "${CMAKE_INSTALL_RELOC_BINDIR}") +# install(TARGETS kim-base64-encode +# RUNTIME DESTINATION "${CMAKE_INSTALL_RELOC_BINDIR}") # Add include subdirectories # diff --git a/cpp/include/KIM_Base64.hpp b/cpp/include/KIM_Base64.hpp deleted file mode 100644 index 8ae84322..00000000 --- a/cpp/include/KIM_Base64.hpp +++ /dev/null @@ -1,216 +0,0 @@ -// ASK RYAN: -// License for this file? The original class was part of Boost, and distributed under -// the Boost Software License. It allows for inclusion of the code in other projects -// so no issues there, but Boost Licence explicitly asks that the Boost License -// must be included along with the LGPL-2.1-or-later license. Is it sufficient to -// include the Boost License in the header file as well, or will it need to be included -// in base project as well? -// Rewrite the class? Though it will be similar to current version with minor changes - -// Currently distributed under Boost Software License - -#ifndef BASE64_HPP -#define BASE64_HPP - -#include -#include -// #include - -namespace Base64 { - // Based on boost https://www.boost.org/doc/libs/1_66_0/boost/beast/core/detail/base64.hpp - namespace - { - /// \brief Base64 alphabet table - inline const char* get_alphabet() { - static const char tab[] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - return tab; - } - - /// \brief Base64 inverse lookup table - inline static const signed char* get_inverse() { - static const signed char tab[] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, - -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, - -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 - }; - return tab; - } - } // anonymous namespace - - /// \brief Calculate encoded size - /// - /// \param n The size of the input - /// \return The size of the encoded output - inline static size_t encoded_size(size_t n) { - return 4 * ((n + 2) / 3); - } - - /// \brief Calculate maximum decoded size - /// - /// \param n The size of the input - /// \return The maximum size of the decoded output - inline static size_t decoded_size(size_t n) { - return n / 4 * 3; - } - - // width of encoded output - const static std::size_t MAX_BASE64_WIDTH = 76; - const static std::size_t MAX_BINARY_WIDTH = MAX_BASE64_WIDTH/4 * 3; - - - /// \brief Encode a string - /// - /// \param input The input string - /// \return The encoded string - inline static std::string encode(const std::string& input) { - std::string output; - output.resize(encoded_size(input.size())); - - const char* in = input.data(); - char* out = &output[0]; - size_t len = input.size(); - const char * tab = get_alphabet(); - - // Process 3-byte chunks - for(size_t n = len / 3; n--;) { - *out++ = tab[(in[0] & 0xfc) >> 2]; - *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; - *out++ = tab[((in[2] & 0xc0) >> 6) + ((in[1] & 0x0f) << 2)]; - *out++ = tab[in[2] & 0x3f]; - in += 3; - } - - // Handle remaining bytes - switch(len % 3) { - case 2: - *out++ = tab[(in[0] & 0xfc) >> 2]; - *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; - *out++ = tab[(in[1] & 0x0f) << 2]; - *out++ = '='; - break; - case 1: - *out++ = tab[(in[0] & 0xfc) >> 2]; - *out++ = tab[(in[0] & 0x03) << 4]; - *out++ = '='; - *out++ = '='; - break; - } - - output.resize(out - &output[0]); - return output; - } - - /// \brief Encode a string, counterpart to encode - /// - /// \param input The input string - /// \return decoded string - inline static std::string decode(const std::string& input) { - std::string output; - output.resize(decoded_size(input.size())); - - const unsigned char* in = reinterpret_cast(input.data()); - char* out = &output[0]; - size_t len = input.size(); - - unsigned char c3[3], c4[4]; - int i = 0; - int j = 0; - - const signed char * inverse = get_inverse(); - - while(len-- && *in != '=') { - const signed char v = inverse[*in]; - if(v == -1) - break; - ++in; - c4[i] = v; - if(++i == 4) { - c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); - c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); - c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; - - for(i = 0; i < 3; i++) - *out++ = c3[i]; - i = 0; - } - } - - if(i) { - c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); - c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); - c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; - - for(j = 0; j < i - 1; j++) - *out++ = c3[j]; - } - - output.resize(out - &output[0]); - return output; - } - - /// \brief Decode a string, but with a more C-like interface - /// - /// \param input pointer to input buffer - /// \param len_in length of the input buffer - /// \param output pointer to store the output buffer - /// \param len_out length of the output buffer written - inline static void decode(unsigned char const * input, - const std::size_t len_in, - unsigned char * const output, - std::size_t& len_out) { - std::size_t len = len_in; - unsigned char* out = output; - unsigned char * in = const_cast(input); - - unsigned char c3[3], c4[4]; - int i = 0; - int j = 0; - - const signed char * inverse = get_inverse(); - - while(len-- && *in != '=') { - const signed char v = inverse[*in]; - if(v == -1) - break; - ++in; - c4[i] = v; - if(++i == 4) { - c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); - c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); - c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; - - for(i = 0; i < 3; i++) - *out++ = c3[i]; - i = 0; - } - } - - if(i) { - c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); - c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); - c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; - - for(j = 0; j < i - 1; j++) - *out++ = c3[j]; - } - - len_out = out - output; - } - -} - -#endif // BASE64_HPP diff --git a/cpp/src/KIM_Base64Encode.cpp b/cpp/src/KIM_Base64Encode.cpp deleted file mode 100644 index e0e903ca..00000000 --- a/cpp/src/KIM_Base64Encode.cpp +++ /dev/null @@ -1,127 +0,0 @@ -// -// Distributed under the existing KIM-API License. -// This is a simple program that reads a file and encodes it in base64 -// and writes it to a new file, like XXD but with base64 encoding - -#include "KIM_Base64.hpp" -#include "KIM_Version.hpp" -#include -#include -#include -#include -#include -#include -#include - -/// \brief Write the Base^4 encode binary file as a C++ source file, emulating XXD -/// -/// \param[in] fileName The name of the file to encode -/// \param[in] outputFileName The name of the file to write the encoded file to -/// \sa Base64::encode -void WriteEncodedFile(std::string & fileName, std::string & outputFileName){ - - std::string xxdFormatFileName = fileName; - // Make the file names compatible with xxd variable name format - std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '.', '_'); - std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '-', '_'); - - // std::ifstream inputFile(fileName, std::ios::binary); - // std::ofstream outputFile(outputFileName); - - FILE *inputFile = fopen(fileName.c_str(), "rb"); - if(!inputFile) { std::cerr << "Error opening input file" << std::endl; return;} - - FILE *outputFile = fopen(outputFileName.c_str(), "w"); - if(!outputFile) {std::cerr << "Error opening output file" << std::endl; return;} - - unsigned int len = 0; - - const size_t chunk = 1024 * 48; // 48 kb buffer, multiple of 3, - // base64 writes 3 bytes to 4 char - std::string buffer; // buffer to store known amount of raw data - buffer.reserve(chunk); - size_t linepos =0 ; - - // C++ const is different from C const (equivalent to static const), - // so we need to use extern - // std::string header = "extern unsigned char " + xxdFormatFileName + "[] = R\"(\n"; - std::string header = "extern unsigned char " + xxdFormatFileName + "[] = \"("; - - // outputFile.write(header.data(), header.length()); - fwrite(header.data(), sizeof(char), header.size(), outputFile); - - std::vector rawBuffer(chunk); // buffer to store raw io data - - // while (inputFile.read(rawBuffer.data(), chunk) || inputFile.gcount()){ - while( size_t bytesRead = fread(rawBuffer.data(), 1, chunk, inputFile) ) - { - buffer.assign(rawBuffer.data(), bytesRead); // assign exact string from raw buffer - // If raw < chunk this will deal with it - - // Possible optimization: stored encoded data in buffer pointer, like the decode function - std::string encoded = Base64::encode(buffer); - for(size_t i = 0; i < encoded.size(); i++){ - char & c = encoded[i]; - // outputFile.put(c); - fputc(c,outputFile); - linepos++; - len++; - if (linepos >= Base64::MAX_BASE64_WIDTH){ - // outputFile.put('\n'); - // fputc('\n',outputFile); - // fwrite("\\n",3,1,outputFile); - len++; - linepos = 0; - } - } - } - // if (linepos > 0) outputFile.put('\n'); - // if (linepos > 0) fputc('\n',outputFile); - // if (linepos > 0) fwrite("\\n",3,1,outputFile); - - - len += 2; // two \n in beginning and end - // std::string footer = ")\";\nextern unsigned int " + xxdFormatFileName + "_len = " + std::to_string(len) + ";\n"; - std::stringstream footerStream; - footerStream << ")\";\nextern unsigned int " << xxdFormatFileName << "_len = " << len << ";\n"; - std::string footer = footerStream.str(); - // outputFile.write(footer.data(), static_cast(footer.length())); - fwrite(footer.data(), sizeof(char), footer.size(), outputFile); - // inputFile.close(); - fclose(inputFile); - // outputFile.close(); - fclose(outputFile); -} - -void usage(std::string name) -{ - size_t beg = name.find_last_of("/\\"); - if (beg != std::string::npos) name = name.substr(beg + 1, std::string::npos); - - // Follows docopt.org format - std::cerr << "Usage:\n" - << " " << name << " " << " " << "\n" - << " " << name << " " << "--version\n"; - // note: this interface is likely to change in future kim-api releases -} - -int main(int argc, char * argv[]){ - - if ((argc == 2) && (std::string(argv[1]) == "--version")) - { - std::cout << KIM_VERSION_STRING << std::endl; - return 0; - } - if ((argc != 3)) - { - usage(argv[0]); - return -1; - } - - - std::string fileName = argv[1]; - std::string outputFileName = argv[2]; - WriteEncodedFile(fileName, outputFileName); - - return 0; -} diff --git a/cpp/src/KIM_SharedLibrary.cpp b/cpp/src/KIM_SharedLibrary.cpp index 839a1cbb..3eafa7c6 100644 --- a/cpp/src/KIM_SharedLibrary.cpp +++ b/cpp/src/KIM_SharedLibrary.cpp @@ -56,7 +56,7 @@ #endif #ifndef BASE64_HPP -#include "KIM_Base64.hpp" // For base64 decoding +#include "base64-encode/base64.hpp" // For base64 decoding #endif namespace @@ -388,10 +388,7 @@ int SharedLibrary::Close() LOG_DEBUG("Exit 1=" + callString); return true; } - else - { - sharedLibraryHandle_ = NULL; - } + else { sharedLibraryHandle_ = NULL; } LOG_DEBUG("Exit 0=" + callString); return false; @@ -653,28 +650,17 @@ int SharedLibrary::WriteParameterFileDirectory() std::ofstream fl; fl.open(specificationFilePathName.string().c_str(), std::ifstream::out | std::ifstream::binary); - const std::size_t FILE_NEWLINE_PAD_OFFSET = 1; - const std::size_t PERLINE_PAD = 1; - - int usable_chars = static_cast(len) - FILE_NEWLINE_PAD_OFFSET*2; - const unsigned char * file_start_ptr = specificationData + FILE_NEWLINE_PAD_OFFSET; - - // std::array binary_line{}; - std::vector binary_line(Base64::MAX_BINARY_WIDTH); - - const int char_per_line = static_cast(Base64::MAX_BASE64_WIDTH); - - int char_remaining = usable_chars; - std::size_t out_len = 0; - unsigned int offset = 0; - - while(char_remaining > 0){ - int n_chars_this_line = std::min(char_per_line, char_remaining); - Base64::decode(file_start_ptr + offset, n_chars_this_line, binary_line.data(), out_len); - fl.write(reinterpret_cast(binary_line.data()), static_cast(out_len)); - char_remaining -= (char_per_line + PERLINE_PAD); - offset += (char_per_line + PERLINE_PAD); - } + + int usable_chars + = static_cast(len); // unsigned int to signed to avoid underflow + + std::vector binary_line; + binary_line.reserve(base64::decoded_size(len)); + std::pair char_out_and_char_in + = base64::decode(binary_line.data(), specificationData, len); + + fl.write(reinterpret_cast(binary_line.data()), + static_cast(char_out_and_char_in.first)); if (!fl) { @@ -704,32 +690,20 @@ int SharedLibrary::WriteParameterFileDirectory() FILESYSTEM::Path const parameterFilePathName = parameterFileDirectoryName_ / parameterFileName; std::ofstream fl; - + fl.open(parameterFilePathName.string().c_str(), std::ifstream::out | std::ifstream::binary); - - const std::size_t FILE_NEWLINE_PAD_OFFSET = 1; - const std::size_t PERLINE_PAD = 1; - - int usable_chars = static_cast(length) - FILE_NEWLINE_PAD_OFFSET*2; - const unsigned char * file_start_ptr = strPtr + FILE_NEWLINE_PAD_OFFSET; - - // std::array binary_line{}; - std::vector binary_line(Base64::MAX_BINARY_WIDTH); - - const int char_per_line = static_cast(Base64::MAX_BASE64_WIDTH); - - int char_remaining = usable_chars; - std::size_t out_len = 0; - unsigned int offset = 0; - - while(char_remaining > 0){ - int n_chars_this_line = std::min(char_per_line, char_remaining); - Base64::decode(file_start_ptr + offset, n_chars_this_line, binary_line.data(), out_len); - fl.write(reinterpret_cast(binary_line.data()), static_cast(out_len)); - char_remaining -= (char_per_line + PERLINE_PAD); - offset += (char_per_line + PERLINE_PAD); - } + + int usable_chars = static_cast( + length); // unsigned int to signed to avoid underflow + + std::vector binary_line; + binary_line.reserve(base64::decoded_size(length)); + std::pair char_out_and_char_in + = base64::decode(binary_line.data(), strPtr, usable_chars); + + fl.write(reinterpret_cast(binary_line.data()), + static_cast(char_out_and_char_in.first)); if (!fl) { LOG_ERROR("Unable to get write parameter file."); diff --git a/cpp/src/base64-encode/CMakeLists.txt b/cpp/src/base64-encode/CMakeLists.txt new file mode 100644 index 00000000..679d09d2 --- /dev/null +++ b/cpp/src/base64-encode/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.10) + +project(base64-encode VERSION 1.0) + +set(CMAKE_CXX_STANDARD 98) +set(CMAKE_CXX_STANDARD_REQUIRED True) +add_compile_options("-std=c++98") # Sometime the CMAKE_CXX_STANDARD does not work + +set(SOURCE_FILES base64-encode.cpp) + +# Add executable target +add_executable(${PROJECT_NAME} ${SOURCE_FILES}) + +set_target_properties(${PROJECT_NAME} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/utils +) + +install(TARGETS ${PROJECT_NAME} DESTINATION bin) diff --git a/cpp/src/base64-encode/LICENSE b/cpp/src/base64-encode/LICENSE new file mode 100644 index 00000000..73fdaa2a --- /dev/null +++ b/cpp/src/base64-encode/LICENSE @@ -0,0 +1,23 @@ +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE \ No newline at end of file diff --git a/cpp/src/base64-encode/README b/cpp/src/base64-encode/README new file mode 100644 index 00000000..62842f9f --- /dev/null +++ b/cpp/src/base64-encode/README @@ -0,0 +1,30 @@ +Base64 Encode +============= + +`base64-encode` is a simple xxd replacement to convert files +into embeddable C++ source code. Instead of using binary array +it used base64 strings, which makes it much more performant +for large files. Currently only -i option is supported for +compatibility. More options to be released in future. C++ 98 compliant. + +### Usage +```bash +base64-encode -i file.in out.cpp +``` +``` +Options: + -h Help + -v Show version information + -i Convert input file to C++ source +``` + +### Build +```bash +mkdir build && cd build +cmake .. +make +sudo make install +``` + +### License +Boost Software license \ No newline at end of file diff --git a/cpp/src/base64-encode/base64-encode.cpp b/cpp/src/base64-encode/base64-encode.cpp new file mode 100644 index 00000000..11d6230e --- /dev/null +++ b/cpp/src/base64-encode/base64-encode.cpp @@ -0,0 +1,134 @@ +// +// Distributed under the Boost Software License 1.0 +// This is a simple program that reads a file and encodes it in base64 +// and writes it to a new file, like XXD but with base64 encoding +// Currently only option -h, -v, -i are supported, more might be added in future +// For any query please contact authors, Navneeth Mohan (nav-mohan) or Amit +// Gupta (ipcamit) + + +#include "base64.hpp" +#include +#include +#include +#include +#include + +#define VERSION "0.1" + +// quick replacement of to_string for C++98 +inline std::string int_to_string(int value) +{ + std::ostringstream oss; + oss << value; + return oss.str(); +} + + +/// \brief Write the Base^4 encode binary file as a C++ source file, emulating +/// XXD +/// +/// \param[in] fileName The name of the file to encode +/// \param[in] outputFileName The name of the file to write the encoded file to +/// \sa base64::encode +void WriteEncodedFile(std::string & fileName, std::string & outputFileName) +{ + unsigned int len = 0; // total written len of all the content + unsigned int linepos + = 0; // current io positions, always < base64::MAX_BASE64_WIDTH + unsigned int n_base64_char; // total base64 char obtained + char rawBuffer[base64::IO_CHUNK]; // buffer to store raw io data + char encodedBuffer[base64::encoded_size( + base64::IO_CHUNK)]; // buffer for converted data + + // setup fine names and pointers + std::string xxdFormatFileName = fileName; + // Make the file names compatible with xxd variable name format + std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '.', '_'); + std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '-', '_'); + + std::ifstream inputFile(fileName.c_str(), std::ios::binary); + std::ofstream outputFile(outputFileName.c_str()); + + + // C++ const is different from C const (equivalent to static const), + // so we need to use extern + std::string header + = "extern unsigned char " + xxdFormatFileName + "[] = \n\""; + outputFile.write(header.data(), header.length()); + + // Read IO_CHUNK, convert to base64 and write to file + while (inputFile.read(rawBuffer, base64::IO_CHUNK) || inputFile.gcount()) + { + n_base64_char + = base64::encode(encodedBuffer, rawBuffer, inputFile.gcount()); + + for (unsigned int i = 0; i < n_base64_char; i++) + { + outputFile.put(encodedBuffer[i]); + linepos++; + len++; + if (linepos >= base64::MAX_BASE64_WIDTH) + { + outputFile.write("\"\n\"", 3); + linepos = 0; + } + } + } + + std::string footer = "\";\nextern unsigned int " + xxdFormatFileName + + "_len = " + int_to_string(len) + ";\n"; + + outputFile.write(footer.data(), + static_cast(footer.length())); + inputFile.close(); + outputFile.close(); +} + + +void print_help() +{ + std::cout + << "Usage: \n\n" + << "base64-encode [OPTIONS] [FILE IN] [FILE OUT]\n\n" + << "base64-encode is a simple xxd replacement to convert files\n" + << "into embeddable C++ source code. Instead of using binary array\n" + << "it used base64 strings, which makes it much more performant \n" + << "for large files. Currently only -i option is supported for \n" + << "compatibility. More options to be released in future.\n" + << "\n" + << "Options:\n" + << " -h Show this help message\n" + << " -v Show version information\n" + << " -i Convert input file to C++ source\n"; +} + + +int main(int argc, char * argv[]) +{ + if ((argc < 2) || (argc > 4)) + { + std::cerr << "IMPROPER ARGUMENTS, please provide input and output file " + "names or options\n"; + print_help(); + return 1; + } + std::string option = argv[1]; + if (option == "-h") { print_help(); } + else if (option == "-v") { std::cout << VERSION << std::endl; } + else if (option == "-i" && argc == 4) + { + std::string fileName = argv[2]; + std::string outputFileName = argv[3]; + WriteEncodedFile(fileName, outputFileName); + } + else + { + std::cerr << "IMPROPER ARGUMENTS, please provide input and output file " + "names or options\n"; + print_help(); + return 1; + } + + return 0; +} diff --git a/cpp/src/base64-encode/base64.hpp b/cpp/src/base64-encode/base64.hpp new file mode 100644 index 00000000..f6cffd94 --- /dev/null +++ b/cpp/src/base64-encode/base64.hpp @@ -0,0 +1,188 @@ +// from https://www.boost.org/doc/libs/1_87_0/boost/beast/core/detail/base64.hpp +// distributed under the same License +#ifndef BASE64_HPP +#define BASE64_HPP + +#include +#include +#include + +namespace base64 +{ + + +/// \brief Returns max chars needed to encode a base64 string +/// +/// \param[in] n Size of char string +/// \return max size of encoded string generated from n chars +inline std::size_t const encoded_size(std::size_t n) +{ + return 4 * ((n + 2) / 3); +} + +/// \brief Returns max bytes needed to decode a base64 string +/// +/// \param[in] n Size of base64 string +/// \return size of char string form n length base64 string +inline std::size_t const decoded_size(std::size_t n) +{ + return n / 4 * 3; // requires n&3==0, smaller +} + +static const int MAX_BASE64_WIDTH = 76; +static const unsigned int IO_CHUNK = 1024 * 48; +static const int MAX_BINARY_WIDTH = static_cast(MAX_BASE64_WIDTH / 4 * 3); + +/// \brief get valid character +/// \return valid char +inline char const * get_alphabet() +{ + static char const tab[] = {"ABCDEFGHIJKLMNOP" + "QRSTUVWXYZabcdef" + "ghijklmnopqrstuv" + "wxyz0123456789+/"}; + return &tab[0]; +} + +inline signed char const * get_inverse() +{ + static signed char const tab[] = { + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, // 0-15 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, // 16-31 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 62, -1, -1, -1, 63, // 32-47 + 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, -1, -1, -1, -1, -1, -1, // 48-63 + -1, 0, 1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14, // 64-79 + 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, -1, -1, -1, -1, -1, // 80-95 + -1, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 36, 37, 38, 39, 40, // 96-111 + 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, -1, -1, -1, -1, -1, // 112-127 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, // 128-143 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, // 144-159 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, // 160-175 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, // 176-191 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, // 192-207 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, // 208-223 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, // 224-239 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1 // 240-255 + }; + return &tab[0]; +} + +/// \brief Encode a series of octets as a padded, base64 string. +/// The resulting string will not be null terminated. +/// The memory pointed to by `out` points to valid memory +/// of at least `encoded_size(len)` bytes. +/// +/// \param[in] dest pointer to location where characters are to be stored +/// \param[in] src pointer to source data +/// \param[in] len length of input string +/// \return The number of characters written to `out`. This +/// will exclude any null termination. +inline std::size_t encode(char * dest, char const * src, std::size_t len) +{ + char * out = dest; + char const * in = src; + const char * tab = base64::get_alphabet(); + + for (size_t n = len / 3; n > 0; n--) + { + *out++ = tab[(in[0] & 0xfc) >> 2]; + *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; + *out++ = tab[((in[2] & 0xc0) >> 6) + ((in[1] & 0x0f) << 2)]; + *out++ = tab[in[2] & 0x3f]; + in += 3; + } + + switch (len % 3) + { + case 2: + *out++ = tab[(in[0] & 0xfc) >> 2]; + *out++ = tab[((in[0] & 0x03) << 4) + ((in[1] & 0xf0) >> 4)]; + *out++ = tab[(in[1] & 0x0f) << 2]; + *out++ = '='; + break; + + case 1: + *out++ = tab[(in[0] & 0xfc) >> 2]; + *out++ = tab[((in[0] & 0x03) << 4)]; + *out++ = '='; + *out++ = '='; + break; + + case 0: break; + } + + return out - static_cast(dest); +} + +/// \brief Decode a padded base64 string into a series of octets. +/// The memory pointed to by `out` points to valid memory +/// of at least `decoded_size(len)` bytes. +/// +/// \param[in] dest +/// \param[in] src +/// \param[in] len +/// \return The number of octets written to `out`, and +/// the number of characters read from the input string, +/// expressed as a pair. +inline std::pair +decode(char * dest, unsigned char const * src, std::size_t len) +{ + char * out = dest; + unsigned char const * in = src; + unsigned char c3[3], c4[4] = {0, 0, 0, 0}; + int i = 0; + int j = 0; + + const signed char * inverse = base64::get_inverse(); + + while (len-- && *in != '=') + { + const signed char v = inverse[*in]; + if (v == -1) break; + ++in; + c4[i] = v; + if (++i == 4) + { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for (i = 0; i < 3; i++) *out++ = c3[i]; + i = 0; + } + } + + if (i) + { + c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4); + c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2); + c3[2] = ((c4[2] & 0x3) << 6) + c4[3]; + + for (j = 0; j < i - 1; j++) *out++ = c3[j]; + } + + std::pair result; + result.first = out - static_cast(dest); + result.second = in - reinterpret_cast(src); + return result; +} + +} // namespace base64 + +#endif // BASE64_HPP From 1ddb0ed0530f993be52477b39fd92c223a992166 Mon Sep 17 00:00:00 2001 From: Amit Gupta Date: Sun, 19 Jan 2025 17:39:16 -0600 Subject: [PATCH 08/14] Working version, modified to remove some compile time warnings + Cmake fix --- CMakeLists.txt | 2 +- cmake/items-macros.cmake.in | 8 ++++---- cpp/src/KIM_SharedLibrary.cpp | 3 --- cpp/src/base64-encode/CMakeLists.txt | 2 +- cpp/src/base64-encode/base64-encode.cpp | 3 +-- cpp/src/base64-encode/base64.hpp | 4 ++-- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a86ec1c..fa56d4f5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -166,7 +166,7 @@ endif() # Keeping the name XXD for now, as otherwise this will result in much larger refactoring # But everywhere, XXD = base64-encode set(XXD_EXECUTABLE ${PROJECT_BINARY_DIR}/utils/base64-encode) -add_subdirectory(cpp/src/base64-encode ${CMAKE_BINARY_DIR}/utils/base64-encode) +add_subdirectory(cpp/src/base64-encode) add_dependencies(kim-api base64-encode) install(TARGETS base64-encode RUNTIME DESTINATION ${CMAKE_INSTALL_RELOC_BINDIR}) diff --git a/cmake/items-macros.cmake.in b/cmake/items-macros.cmake.in index 3ccd08db..41240961 100644 --- a/cmake/items-macros.cmake.in +++ b/cmake/items-macros.cmake.in @@ -503,9 +503,9 @@ endmacro() # function(_add_custom_command_blob_to_cpp _dirin _filein _fileout) include(FindPackageMessage) - find_program(XXD_EXECUTABLE "kim-base64-encode") + find_program(XXD_EXECUTABLE "base64-encode") if(XXD_EXECUTABLE) - find_package_message(xxd "Found kim-base64-encode: (${XXD_EXECUTABLE})" "found") + find_package_message(xxd "Found base64-encode: (${XXD_EXECUTABLE})" "found") string(MAKE_C_IDENTIFIER ${_filein} _cfilein) set(_edit_xxd_output "${CMAKE_CURRENT_BINARY_DIR}/EditXXDOutput.cmake") @@ -522,7 +522,7 @@ function(_add_custom_command_blob_to_cpp _dirin _filein _fileout) ) endif() add_custom_command(OUTPUT ${_fileout} - COMMAND ${XXD_EXECUTABLE} "${_filein}" "${_fileout}" + COMMAND ${XXD_EXECUTABLE} -i "${_filein}" "${_fileout}" COMMAND ${CMAKE_COMMAND} "-Dvarname=${_cfilein}" "-Dfilein=${_filein}" "-Dfileout=${_fileout}" -P "${_edit_xxd_output}" DEPENDS "${_dirin}/${_filein}" "${_edit_xxd_output}" WORKING_DIRECTORY "${_dirin}" @@ -533,7 +533,7 @@ function(_add_custom_command_blob_to_cpp _dirin _filein _fileout) # No backup yet # Possible Solutions: 1) CMAKE only implementation of base64 encoding? # 2) use system base64/openssl binaries? - message(FATAL_ERROR "Missing kim-base64-encode: Please check the KIM-API installation.") + message(FATAL_ERROR "Missing base64-encode: Please check the KIM-API installation.") # find_package_message(xxd "Missing xxd: Falling back to less efficient cmake implementation." "missing") # string(MAKE_C_IDENTIFIER ${_filein} _cfilein) diff --git a/cpp/src/KIM_SharedLibrary.cpp b/cpp/src/KIM_SharedLibrary.cpp index 3eafa7c6..76770f29 100644 --- a/cpp/src/KIM_SharedLibrary.cpp +++ b/cpp/src/KIM_SharedLibrary.cpp @@ -651,9 +651,6 @@ int SharedLibrary::WriteParameterFileDirectory() fl.open(specificationFilePathName.string().c_str(), std::ifstream::out | std::ifstream::binary); - int usable_chars - = static_cast(len); // unsigned int to signed to avoid underflow - std::vector binary_line; binary_line.reserve(base64::decoded_size(len)); std::pair char_out_and_char_in diff --git a/cpp/src/base64-encode/CMakeLists.txt b/cpp/src/base64-encode/CMakeLists.txt index 679d09d2..6c3d6611 100644 --- a/cpp/src/base64-encode/CMakeLists.txt +++ b/cpp/src/base64-encode/CMakeLists.txt @@ -13,6 +13,6 @@ add_executable(${PROJECT_NAME} ${SOURCE_FILES}) set_target_properties(${PROJECT_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/utils -) + ) install(TARGETS ${PROJECT_NAME} DESTINATION bin) diff --git a/cpp/src/base64-encode/base64-encode.cpp b/cpp/src/base64-encode/base64-encode.cpp index 11d6230e..1f05822f 100644 --- a/cpp/src/base64-encode/base64-encode.cpp +++ b/cpp/src/base64-encode/base64-encode.cpp @@ -38,8 +38,7 @@ void WriteEncodedFile(std::string & fileName, std::string & outputFileName) = 0; // current io positions, always < base64::MAX_BASE64_WIDTH unsigned int n_base64_char; // total base64 char obtained char rawBuffer[base64::IO_CHUNK]; // buffer to store raw io data - char encodedBuffer[base64::encoded_size( - base64::IO_CHUNK)]; // buffer for converted data + char encodedBuffer[4 * ((base64::IO_CHUNK + 2) / 3)]; // buffer for converted data // setup fine names and pointers std::string xxdFormatFileName = fileName; diff --git a/cpp/src/base64-encode/base64.hpp b/cpp/src/base64-encode/base64.hpp index f6cffd94..0ba59906 100644 --- a/cpp/src/base64-encode/base64.hpp +++ b/cpp/src/base64-encode/base64.hpp @@ -15,7 +15,7 @@ namespace base64 /// /// \param[in] n Size of char string /// \return max size of encoded string generated from n chars -inline std::size_t const encoded_size(std::size_t n) +inline std::size_t encoded_size(std::size_t n) { return 4 * ((n + 2) / 3); } @@ -24,7 +24,7 @@ inline std::size_t const encoded_size(std::size_t n) /// /// \param[in] n Size of base64 string /// \return size of char string form n length base64 string -inline std::size_t const decoded_size(std::size_t n) +inline std::size_t decoded_size(std::size_t n) { return n / 4 * 3; // requires n&3==0, smaller } From 27200f5f224d2fa0480001a2aa5f2aefe6223c25 Mon Sep 17 00:00:00 2001 From: "Ryan S. Elliott" Date: Wed, 22 Jan 2025 16:48:16 -0500 Subject: [PATCH 09/14] Ryan's work on refactoring base64 additions --- CMakeLists.txt | 18 --- cmake/build-tree-config.cmake.in | 1 + cmake/items-macros.cmake.in | 103 +++++++--------- cpp/src/KIM_SharedLibrary.cpp | 8 +- cpp/src/base64-encode/base64-encode.cpp | 133 --------------------- utils/CMakeLists.txt | 8 +- utils/base64-encode.cpp | 149 ++++++++++++++++++++++++ 7 files changed, 200 insertions(+), 220 deletions(-) delete mode 100644 cpp/src/base64-encode/base64-encode.cpp create mode 100644 utils/base64-encode.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fa56d4f5..66492719 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,12 +47,6 @@ else() set(_LOG_MAX "INFORMATION") set(CMAKE_BUILD_TYPE "Release") # Default to Release, solves KIM slowdown complaints endif() - -message(STATUS "KIM-API Build Type: ${CMAKE_BUILD_TYPE}") - -# C++ Language Standard, enforced to be C++11 -# set(CMAKE_CXX_STANDARD 11) - set_cache_with_fallback(KIM_API_LOG_MAXIMUM_LEVEL "${_LOG_MAX}" STRING "Maximum log verbosity") unset(_LOG_MAX) set_property(CACHE KIM_API_LOG_MAXIMUM_LEVEL PROPERTY STRINGS "" SILENT FATAL ERROR WARNING INFORMATION DEBUG) @@ -162,15 +156,6 @@ if(NOT WIN32 OR CYGWIN) target_link_libraries(kim-api ${CMAKE_DL_LIBS}) endif() - -# Keeping the name XXD for now, as otherwise this will result in much larger refactoring -# But everywhere, XXD = base64-encode -set(XXD_EXECUTABLE ${PROJECT_BINARY_DIR}/utils/base64-encode) -add_subdirectory(cpp/src/base64-encode) -add_dependencies(kim-api base64-encode) -install(TARGETS base64-encode - RUNTIME DESTINATION ${CMAKE_INSTALL_RELOC_BINDIR}) - # Add install rules for kim-api # install(TARGETS kim-api @@ -191,9 +176,6 @@ install( DESTINATION "${CMAKE_INSTALL_RELOC_DOCDIR}" ) -# install kim-base64-encode in CMAKE_INSTALL_RELOC_BINDIR -# install(TARGETS kim-base64-encode -# RUNTIME DESTINATION "${CMAKE_INSTALL_RELOC_BINDIR}") # Add include subdirectories # diff --git a/cmake/build-tree-config.cmake.in b/cmake/build-tree-config.cmake.in index 425aa7e1..5b31fbd3 100644 --- a/cmake/build-tree-config.cmake.in +++ b/cmake/build-tree-config.cmake.in @@ -42,6 +42,7 @@ if("${KIM-API_FIND_VERSION}" VERSION_GREATER "2.1.99") add_executable(KIM-API::portable-model-info ALIAS portable-model-info) add_executable(KIM-API::simulator-model ALIAS simulator-model) add_executable(KIM-API::collections-info ALIAS collections-info) + add_executable(KIM-API::base64-encode ALIAS base64-encode) add_executable(KIM-API::shared-library-test ALIAS shared-library-test) endif() diff --git a/cmake/items-macros.cmake.in b/cmake/items-macros.cmake.in index 41240961..a5a7f7a0 100644 --- a/cmake/items-macros.cmake.in +++ b/cmake/items-macros.cmake.in @@ -502,63 +502,44 @@ endmacro() # Sets _blob_to_c_command to string for use with add_custom_command() # function(_add_custom_command_blob_to_cpp _dirin _filein _fileout) - include(FindPackageMessage) - find_program(XXD_EXECUTABLE "base64-encode") - if(XXD_EXECUTABLE) - find_package_message(xxd "Found base64-encode: (${XXD_EXECUTABLE})" "found") - - string(MAKE_C_IDENTIFIER ${_filein} _cfilein) - set(_edit_xxd_output "${CMAKE_CURRENT_BINARY_DIR}/EditXXDOutput.cmake") - if(NOT EXISTS "${_edit_xxd_output}") - file(WRITE "${_edit_xxd_output}" # use a bracket argument to avoid ugly escaping - [=[ - # This file was automatically generated by CMake; do not edit. - file(READ "${fileout}" _content) - string(REGEX REPLACE "unsigned char.*[[][]]" "unsigned char const ${varname}[]" _content "${_content}") - string(REGEX REPLACE "unsigned int.*=" "unsigned int const ${varname}_len =" _content "${_content}") - file(WRITE "${fileout}" "${_content}") - unset(_content) - ]=] - ) + get_property(_IMPORTED TARGET KIM-API::base64-encode PROPERTY "IMPORTED") + if(_IMPORTED) # using install tree config + if("${CMAKE_BUILD_TYPE}" STREQUAL "") + set(_BUILD_TYPE "NOCONFIG") + else() + string(TOUPPER "${CMAKE_BUILD_TYPE}" _BUILD_TYPE) endif() - add_custom_command(OUTPUT ${_fileout} - COMMAND ${XXD_EXECUTABLE} -i "${_filein}" "${_fileout}" - COMMAND ${CMAKE_COMMAND} "-Dvarname=${_cfilein}" "-Dfilein=${_filein}" "-Dfileout=${_fileout}" -P "${_edit_xxd_output}" - DEPENDS "${_dirin}/${_filein}" "${_edit_xxd_output}" - WORKING_DIRECTORY "${_dirin}" - ) - unset(_cfilein) - unset(_edit_xxd_output) - else() - # No backup yet - # Possible Solutions: 1) CMAKE only implementation of base64 encoding? - # 2) use system base64/openssl binaries? - message(FATAL_ERROR "Missing base64-encode: Please check the KIM-API installation.") - # find_package_message(xxd "Missing xxd: Falling back to less efficient cmake implementation." "missing") - - # string(MAKE_C_IDENTIFIER ${_filein} _cfilein) - # set(_blob_to_source "${CMAKE_CURRENT_BINARY_DIR}/BlobToCSource.cmake") - # if(NOT EXISTS "${_blob_to_source}") - # file(WRITE "${_blob_to_source}" # use a bracket argument to avoid ugly escaping - # [=[ - # # This file was automatically generated by CMake; do not edit. - # file(READ "${filein}" _content HEX) - # string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," _content "${_content}") - # string(REGEX REPLACE ",$" "" _content "${_content}") - # file(WRITE "${fileout}" "unsigned char const ${varname}[] = { ${_content} };\n") - # file(APPEND "${fileout}" "unsigned int const ${varname}_len = sizeof( ${varname} );\n") - # unset(_content) - # ]=] - # ) - # endif() - # add_custom_command(OUTPUT ${_fileout} - # COMMAND ${CMAKE_COMMAND} "-Dvarname=${_cfilein}" "-Dfilein=${_filein}" "-Dfileout=${_fileout}" -P "${_blob_to_source}" - # DEPENDS "${_dirin}/${_filein}" "${_blob_to_source}" - # WORKING_DIRECTORY "${_dirin}" - # ) - # unset(_cfilein) - # unset(_blob_to_source) + get_property(_BASE64_ENCODE TARGET KIM-API::base64-encode PROPERTY "IMPORTED_LOCATION_${_BUILD_TYPE}") + unset(_BUILD_TYPE) + if("${_BASE64_ENCODE}" STREQUAL "") + message(FATAL_ERROR "Unable to locate 'base64-encode' utility") + endif() + else() # using build tree config + set(_BASE64_ENCODE KIM-API::base64-encode) + endif() + string(MAKE_C_IDENTIFIER ${_filein} _cfilein) + set(_edit_encode_output "${CMAKE_CURRENT_BINARY_DIR}/EditEncodeOutput.cmake") + if(NOT EXISTS "${_edit_encode_output}") + file(WRITE "${_edit_encode_output}" # use a bracket argument to avoid ugly escaping + [=[ + # This file was automatically generated by CMake; do not edit. + file(READ "${fileout}" _content) + string(REGEX REPLACE "unsigned char.*[[][]]" "unsigned char const ${varname}[]" _content "${_content}") + string(REGEX REPLACE "unsigned int.*=" "unsigned int const ${varname}_len =" _content "${_content}") + file(WRITE "${fileout}" "${_content}") + unset(_content) + ]=] + ) endif() + add_custom_command(OUTPUT ${_fileout} + COMMAND ${_BASE64_ENCODE} -i "${_filein}" "${_fileout}" + COMMAND ${CMAKE_COMMAND} "-Dvarname=${_cfilein}" "-Dfilein=${_filein}" "-Dfileout=${_fileout}" -P "${_edit_encode_output}" + DEPENDS "${_dirin}/${_filein}" "${_edit_encode_output}" + WORKING_DIRECTORY "${_dirin}" + ) + unset(_cfilein) + unset(_edit_encode_output) + unset(_BASE64_ENCODE) endfunction(_add_custom_command_blob_to_cpp) # @@ -568,13 +549,13 @@ endfunction(_add_custom_command_blob_to_cpp) # Sets ITEM_*_SOURCES, ITEM_*_DECLARATIONS, ITEM_*_EMBEDDED_FILE_IDENTIFIER, # and ITEM_*_EMBEDDED_FILE_INITIALIZER_LIST variables in parent scope. # -function(_xxd_process_files) +function(_encode_process_files) set(_options "") set(_oneValueArgs FILE_TYPE) set(_multiValueArgs FILE_NAMES) cmake_parse_arguments(_ITEM "${_options}" "${_oneValueArgs}" "${_multiValueArgs}" ${ARGN}) if(_ITEM_UNPARSED_ARGUMENTS) - message(FATAL_ERROR "Unparsed arguments found in macro '_xxd_process_files'") + message(FATAL_ERROR "Unparsed arguments found in macro '_encode_process_files'") endif() unset(_options) unset(_oneValueArgs) @@ -624,7 +605,7 @@ function(_xxd_process_files) set(ITEM_${_ITEM_FILE_TYPE}_DECLARATIONS "${_ITEM_DECLARATIONS}" PARENT_SCOPE) set(ITEM_${_ITEM_FILE_TYPE}_EMBEDDED_FILE_IDENTIFIER "${_ITEM_EMBEDDED_FILE_IDENTIFIER}" PARENT_SCOPE) set(ITEM_${_ITEM_FILE_TYPE}_EMBEDDED_FILE_INITIALIZER_LIST "${_ITEM_EMBEDDED_FILE_INITIALIZER_LIST}" PARENT_SCOPE) -endfunction(_xxd_process_files) +endfunction(_encode_process_files) # # Function to set collection install prefix and item library file name. @@ -797,11 +778,11 @@ function(_add_kim_api_library) endif() set(_ITEM_SOURCES "") - _xxd_process_files(FILE_TYPE SM_SPEC_FILE FILE_NAMES "${ITEM_SM_SPEC_FILE}") + _encode_process_files(FILE_TYPE SM_SPEC_FILE FILE_NAMES "${ITEM_SM_SPEC_FILE}") list(APPEND _ITEM_SOURCES ${ITEM_SM_SPEC_FILE_SOURCES}) - _xxd_process_files(FILE_TYPE PARAMETER_FILE FILE_NAMES "${ITEM_PARAMETER_FILES}") + _encode_process_files(FILE_TYPE PARAMETER_FILE FILE_NAMES "${ITEM_PARAMETER_FILES}") list(APPEND _ITEM_SOURCES ${ITEM_PARAMETER_FILE_SOURCES}) - _xxd_process_files(FILE_TYPE METADATA_FILE FILE_NAMES "${ITEM_METADATA_FILES}") + _encode_process_files(FILE_TYPE METADATA_FILE FILE_NAMES "${ITEM_METADATA_FILES}") list(APPEND _ITEM_SOURCES ${ITEM_METADATA_FILE_SOURCES}) configure_file(${KIM-API-ITEMS_DIR}/item-info.txt.in ${CMAKE_CURRENT_BINARY_DIR}/item-info.txt @ONLY) diff --git a/cpp/src/KIM_SharedLibrary.cpp b/cpp/src/KIM_SharedLibrary.cpp index 76770f29..02d30731 100644 --- a/cpp/src/KIM_SharedLibrary.cpp +++ b/cpp/src/KIM_SharedLibrary.cpp @@ -28,6 +28,7 @@ // Release: This file is part of the kim-api.git repository. // +#include "base64-encode/base64.hpp" // For base64 decoding #include #include #ifndef _WIN32 @@ -55,10 +56,6 @@ #include "KIM_SharedLibrarySchema.hpp" #endif -#ifndef BASE64_HPP -#include "base64-encode/base64.hpp" // For base64 decoding -#endif - namespace { static void * const referencePointForKIM_Library = NULL; @@ -650,7 +647,6 @@ int SharedLibrary::WriteParameterFileDirectory() std::ofstream fl; fl.open(specificationFilePathName.string().c_str(), std::ifstream::out | std::ifstream::binary); - std::vector binary_line; binary_line.reserve(base64::decoded_size(len)); std::pair char_out_and_char_in @@ -687,10 +683,8 @@ int SharedLibrary::WriteParameterFileDirectory() FILESYSTEM::Path const parameterFilePathName = parameterFileDirectoryName_ / parameterFileName; std::ofstream fl; - fl.open(parameterFilePathName.string().c_str(), std::ifstream::out | std::ifstream::binary); - int usable_chars = static_cast( length); // unsigned int to signed to avoid underflow diff --git a/cpp/src/base64-encode/base64-encode.cpp b/cpp/src/base64-encode/base64-encode.cpp deleted file mode 100644 index 1f05822f..00000000 --- a/cpp/src/base64-encode/base64-encode.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// -// Distributed under the Boost Software License 1.0 -// This is a simple program that reads a file and encodes it in base64 -// and writes it to a new file, like XXD but with base64 encoding -// Currently only option -h, -v, -i are supported, more might be added in future -// For any query please contact authors, Navneeth Mohan (nav-mohan) or Amit -// Gupta (ipcamit) - - -#include "base64.hpp" -#include -#include -#include -#include -#include - -#define VERSION "0.1" - -// quick replacement of to_string for C++98 -inline std::string int_to_string(int value) -{ - std::ostringstream oss; - oss << value; - return oss.str(); -} - - -/// \brief Write the Base^4 encode binary file as a C++ source file, emulating -/// XXD -/// -/// \param[in] fileName The name of the file to encode -/// \param[in] outputFileName The name of the file to write the encoded file to -/// \sa base64::encode -void WriteEncodedFile(std::string & fileName, std::string & outputFileName) -{ - unsigned int len = 0; // total written len of all the content - unsigned int linepos - = 0; // current io positions, always < base64::MAX_BASE64_WIDTH - unsigned int n_base64_char; // total base64 char obtained - char rawBuffer[base64::IO_CHUNK]; // buffer to store raw io data - char encodedBuffer[4 * ((base64::IO_CHUNK + 2) / 3)]; // buffer for converted data - - // setup fine names and pointers - std::string xxdFormatFileName = fileName; - // Make the file names compatible with xxd variable name format - std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '.', '_'); - std::replace(xxdFormatFileName.begin(), xxdFormatFileName.end(), '-', '_'); - - std::ifstream inputFile(fileName.c_str(), std::ios::binary); - std::ofstream outputFile(outputFileName.c_str()); - - - // C++ const is different from C const (equivalent to static const), - // so we need to use extern - std::string header - = "extern unsigned char " + xxdFormatFileName + "[] = \n\""; - outputFile.write(header.data(), header.length()); - - // Read IO_CHUNK, convert to base64 and write to file - while (inputFile.read(rawBuffer, base64::IO_CHUNK) || inputFile.gcount()) - { - n_base64_char - = base64::encode(encodedBuffer, rawBuffer, inputFile.gcount()); - - for (unsigned int i = 0; i < n_base64_char; i++) - { - outputFile.put(encodedBuffer[i]); - linepos++; - len++; - if (linepos >= base64::MAX_BASE64_WIDTH) - { - outputFile.write("\"\n\"", 3); - linepos = 0; - } - } - } - - std::string footer = "\";\nextern unsigned int " + xxdFormatFileName - + "_len = " + int_to_string(len) + ";\n"; - - outputFile.write(footer.data(), - static_cast(footer.length())); - inputFile.close(); - outputFile.close(); -} - - -void print_help() -{ - std::cout - << "Usage: \n\n" - << "base64-encode [OPTIONS] [FILE IN] [FILE OUT]\n\n" - << "base64-encode is a simple xxd replacement to convert files\n" - << "into embeddable C++ source code. Instead of using binary array\n" - << "it used base64 strings, which makes it much more performant \n" - << "for large files. Currently only -i option is supported for \n" - << "compatibility. More options to be released in future.\n" - << "\n" - << "Options:\n" - << " -h Show this help message\n" - << " -v Show version information\n" - << " -i Convert input file to C++ source\n"; -} - - -int main(int argc, char * argv[]) -{ - if ((argc < 2) || (argc > 4)) - { - std::cerr << "IMPROPER ARGUMENTS, please provide input and output file " - "names or options\n"; - print_help(); - return 1; - } - std::string option = argv[1]; - if (option == "-h") { print_help(); } - else if (option == "-v") { std::cout << VERSION << std::endl; } - else if (option == "-i" && argc == 4) - { - std::string fileName = argv[2]; - std::string outputFileName = argv[3]; - WriteEncodedFile(fileName, outputFileName); - } - else - { - std::cerr << "IMPROPER ARGUMENTS, please provide input and output file " - "names or options\n"; - print_help(); - return 1; - } - - return 0; -} diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index e625cd05..6e1b8284 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -35,6 +35,7 @@ include(RelocatablePath) set(PORTABLE_MODEL_INFO ${PROJECT_NAME}-portable-model-info) set(SIMULATOR_MODEL ${PROJECT_NAME}-simulator-model) set(COLLECTIONS_INFO ${PROJECT_NAME}-collections-info) +set(BASE64_ENCODE ${PROJECT_NAME}-base64-encode) set(COLLECTIONS_MANAGEMENT ${PROJECT_NAME}-collections-management) set(SHARED_LIBRARY_TEST ${PROJECT_NAME}-shared-library-test) set(ACTIVATE_SCRIPT ${PROJECT_NAME}-activate) @@ -126,6 +127,11 @@ target_include_directories(collections-info PRIVATE "${PROJECT_SOURCE_DIR}/cpp/s target_link_libraries(collections-info kim-api) set_target_properties(collections-info PROPERTIES OUTPUT_NAME ${COLLECTIONS_INFO}) +add_executable(base64-encode base64-encode.cpp) +target_include_directories(base64-encode PRIVATE "${PROJECT_SOURCE_DIR}/cpp/src/") +target_link_libraries(base64-encode kim-api) +set_target_properties(base64-encode PROPERTIES OUTPUT_NAME ${BASE64_ENCODE}) + add_executable(shared-library-test shared-library-test.cpp) target_include_directories(shared-library-test PRIVATE "${PROJECT_BINARY_DIR}/cpp/include") target_link_libraries(shared-library-test ${CMAKE_DL_LIBS}) @@ -138,7 +144,7 @@ else() set(_dir "${CMAKE_INSTALL_RELOC_BINDIR}") endif() install( - TARGETS portable-model-info simulator-model collections-info shared-library-test + TARGETS portable-model-info simulator-model collections-info base64-encode shared-library-test EXPORT KIM_API_Targets DESTINATION "${_dir}" ) diff --git a/utils/base64-encode.cpp b/utils/base64-encode.cpp new file mode 100644 index 00000000..0a2f7066 --- /dev/null +++ b/utils/base64-encode.cpp @@ -0,0 +1,149 @@ +// +// KIM-API: An API for interatomic models +// Copyright (c) 2013--2022, Regents of the University of Minnesota. +// All rights reserved. +// +// Contributors: +// Navaneeth Mohan +// Amit Gupta +// Ryan S. Elliott +// +// SPDX-License-Identifier: LGPL-2.1-or-later +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this library; if not, write to the Free Software Foundation, +// Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +// +// Release: This file is part of the kim-api.git repository. +// + + +#include "KIM_Version.hpp" +#include "base64-encode/base64.hpp" +#include +#include +#include +#include +#include + +void usage(std::string name) +{ + size_t beg = name.find_last_of("/\\"); + if (beg != std::string::npos) name = name.substr(beg + 1, std::string::npos); + + // Follows docopt.org format + std::cerr << "Usage:\n" + << " " << name << " -i \n" + << " " << name << "--version\n"; + // note: this interface is likely to change in future kim-api releases +} + + +namespace B64ENCODE +{ +// quick replacement of to_string for C++98 +inline std::string int_to_string(int value) +{ + std::ostringstream oss; + oss << value; + return oss.str(); +} + +/// \brief Write the Base^4 encode binary file as a C++ source file, emulating +/// XXD +/// +/// \param[in] fileName The name of the file to encode +/// \param[in] outputFileName The name of the file to write the encoded file to +/// \sa base64::encode +void WriteEncodedFile(std::string & fileName, std::string & outputFileName) +{ + unsigned int len = 0; // total written len of all the content + unsigned int linepos + = 0; // current io positions, always < base64::MAX_BASE64_WIDTH + unsigned int n_base64_char; // total base64 char obtained + char rawBuffer[base64::IO_CHUNK]; // buffer to store raw io data + char encodedBuffer[4 * ((base64::IO_CHUNK + 2) / 3)]; // buffer for converted + // data + + // setup fine names and pointers + std::string encodeFormatFileName = fileName; + // Make the file names compatible with encode variable name format + std::replace( + encodeFormatFileName.begin(), encodeFormatFileName.end(), '.', '_'); + std::replace( + encodeFormatFileName.begin(), encodeFormatFileName.end(), '-', '_'); + + std::ifstream inputFile(fileName.c_str(), std::ios::binary); + std::ofstream outputFile(outputFileName.c_str()); + + + // C++ const is different from C const (equivalent to static const), + // so we need to use extern + std::string header + = "extern unsigned char " + encodeFormatFileName + "[] = \n\""; + outputFile.write(header.data(), header.length()); + + // Read IO_CHUNK, convert to base64 and write to file + while (inputFile.read(rawBuffer, base64::IO_CHUNK) || inputFile.gcount()) + { + n_base64_char + = base64::encode(encodedBuffer, rawBuffer, inputFile.gcount()); + + for (unsigned int i = 0; i < n_base64_char; i++) + { + outputFile.put(encodedBuffer[i]); + linepos++; + len++; + if (linepos >= base64::MAX_BASE64_WIDTH) + { + outputFile.write("\"\n\"", 3); + linepos = 0; + } + } + } + + std::string footer = "\";\nextern unsigned int " + encodeFormatFileName + + "_len = " + int_to_string(len) + ";\n"; + + outputFile.write(footer.data(), + static_cast(footer.length())); + inputFile.close(); + outputFile.close(); +} +} // namespace B64ENCODE + +int main(int argc, char * argv[]) +{ + if ((argc < 2) || (argc > 4)) + { + usage(argv[0]); + return 1; + } + std::string option = argv[1]; + if (option == "--version") { std::cout << KIM_VERSION_STRING << std::endl; } + else if (option == "-i" && argc == 4) + { + std::string fileName = argv[2]; + std::string outputFileName = argv[3]; + B64ENCODE::WriteEncodedFile(fileName, outputFileName); + } + else + { + usage(argv[0]); + return 1; + } + + return 0; +} From 746172b91a4960c0c632414f6b67a2acd3976287 Mon Sep 17 00:00:00 2001 From: nav-mohan Date: Tue, 21 Jan 2025 19:24:45 -0500 Subject: [PATCH 10/14] Trigger build-workflow manually This will enable the "Run Workflow" button for our Github Actions on the Github Web Interface. --- .github/workflows/build-multiplatform.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-multiplatform.yml b/.github/workflows/build-multiplatform.yml index 669081dd..b56c6937 100644 --- a/.github/workflows/build-multiplatform.yml +++ b/.github/workflows/build-multiplatform.yml @@ -41,6 +41,7 @@ on: branches-ignore: - 'coverity_scan' pull_request: + workflow_dispatch: # Trigger workflow manually through Github API, Github CLI, or Github Web-UI jobs: From b2b91fed6c95e22f1a2797388b2dbb2bfd1486da Mon Sep 17 00:00:00 2001 From: nav-mohan Date: Tue, 21 Jan 2025 19:25:44 -0500 Subject: [PATCH 11/14] Explicit installation of fortran for Windows-MinGW The Fortran compiler was not included in the installation of mingw-w64-x86_64-toolchain. It must be explicitly installed through mingw-w64-x86_64-gcc-fortran --- .github/workflows/build-multiplatform.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-multiplatform.yml b/.github/workflows/build-multiplatform.yml index b56c6937..d389d301 100644 --- a/.github/workflows/build-multiplatform.yml +++ b/.github/workflows/build-multiplatform.yml @@ -336,6 +336,7 @@ jobs: mingw-w64-x86_64-cmake mingw-w64-x86_64-doxygen mingw-w64-x86_64-toolchain + mingw-w64-x86_64-gcc-fortran - name: Ensure windows ccache dir exists run: | @@ -355,6 +356,7 @@ jobs: - name: Build KIM API, run tests env: + FC: gfortran CMAKE_GENERATOR: MinGW Makefiles MAKE: mingw32-make CMAKE_MAKE_PROGRAM: mingw32-make From e7bfdba5153179f232bd6f3d1544707025a64dfa Mon Sep 17 00:00:00 2001 From: nav-mohan Date: Tue, 21 Jan 2025 19:58:58 -0500 Subject: [PATCH 12/14] debian upgrade libasan6 to libasan8 --- docker/Dockerfile.debian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.debian b/docker/Dockerfile.debian index 187b60b3..6f0ad280 100644 --- a/docker/Dockerfile.debian +++ b/docker/Dockerfile.debian @@ -41,7 +41,7 @@ RUN apt-get -y update && \ gfortran \ git \ iwyu \ - libasan6 \ + libasan8 \ make \ wget \ xxd \ From 78c1bfa048478fd80a1808215d233142c5b3c990 Mon Sep 17 00:00:00 2001 From: nav-mohan Date: Tue, 21 Jan 2025 20:05:01 -0500 Subject: [PATCH 13/14] opensuse upgrade libasan6 to libasan8 --- docker/Dockerfile.openSUSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.openSUSE b/docker/Dockerfile.openSUSE index 497b54b3..7045f362 100644 --- a/docker/Dockerfile.openSUSE +++ b/docker/Dockerfile.openSUSE @@ -41,7 +41,7 @@ RUN zypper -n install \ gcc-c++ \ gcc-fortran \ include-what-you-use \ - libasan6 \ + libasan8 \ make \ sysuser-shadow \ tar \ From d9e7f520c6bbdf9c3f12052e80931d02d2af063a Mon Sep 17 00:00:00 2001 From: nav-mohan Date: Tue, 21 Jan 2025 20:51:29 -0500 Subject: [PATCH 14/14] Upgrade MacOS from 10,11 to 13,14 --- .github/workflows/build-multiplatform.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-multiplatform.yml b/.github/workflows/build-multiplatform.yml index d389d301..fa232f85 100644 --- a/.github/workflows/build-multiplatform.yml +++ b/.github/workflows/build-multiplatform.yml @@ -50,15 +50,15 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-10.15, macos-11] + os: [macos-13-large, macos-14-large] include: - - os: macos-10.15 - DISTRO: macos_catalina - job_name: macOS Catalina 10.15 (debug) + - os: macos-13-large + DISTRO: macos_ventura + job_name: macOS Ventura 13 (debug) - - os: macos-11 - DISTRO: macos_bigsur - job_name: macOS Big Sur 11 (debug) + - os: macos-14-large + DISTRO: macos_sonoma + job_name: macOS Sonoma 14 (debug) name: ${{ matrix.job_name }}