diff --git a/.cirrus.yml b/.cirrus.yml index 43d986b1b..7f9a2f59f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -70,9 +70,7 @@ task: brew update brew install openssl@3 \ bison flex gnu-sed bash boost double-conversion jemalloc fmt glog pkg-config \ - libevent libsodium lz4 xz zlib - -# No ccache - it causes '/bin/sh: fork: Resource temporarily unavailable' in 90% of runs + libevent libsodium lz4 xz zlib ccache pre_configure_script: echo CXX_FLAGS='-DTARGET_OS_SIMULATOR=0 -DTARGET_OS_IPHONE=0' >> $CIRRUS_ENV diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ac5e6eda..c7eea99f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,20 +100,6 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") add_compile_options(/Zc:__cplusplus /utf-8 /wd4267 /wd4244 /wd5219) endif() -if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") - set(CMAKE_C_LINK_LIBRARY_USING_WHOLE_ARCHIVE "-force_load ") -elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(CMAKE_C_LINK_LIBRARY_USING_WHOLE_ARCHIVE "/WHOLEARCHIVE:") -elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(CMAKE_C_LINK_LIBRARY_USING_WHOLE_ARCHIVE - "LINKER:--push-state,--whole-archive" - "" - "LINKER:--pop-state" - ) -else() - message(FATAL_ERROR "WHOLE_ARCHIVE is unsupported on this platform") -endif() - include(${CMAKE_SOURCE_DIR}/cmake/version.cmake) # ----------------------------------------------------------------------------------------- @@ -608,28 +594,27 @@ if("${TEBAKO_BUILD_SCOPE}" STREQUAL "ALL") target_link_libraries(dwarfs_main ${WINFSP}) target_link_libraries(dwarfsuniversal delayimp.lib) target_link_options(dwarfsuniversal PRIVATE /DELAYLOAD:winfsp-x64.dll) - else() + else(WINFSP) target_compile_definitions(dwarfs_main PRIVATE FUSE_USE_VERSION=35) target_link_libraries(dwarfs_main PkgConfig::FUSE3) - endif() + endif(WINFSP) add_executable(dwarfs-bin src/dwarfs.cpp) target_link_libraries(dwarfs-bin dwarfs_main) set_target_properties(dwarfs-bin PROPERTIES OUTPUT_NAME dwarfs) target_link_libraries(dwarfsuniversal dwarfs_main) if(WINFSP) install(TARGETS dwarfs-bin RUNTIME DESTINATION bin) - else() + else(WINFSP) add_custom_command(OUTPUT mount.dwarfs COMMAND ${CMAKE_COMMAND} -E create_symlink dwarfs mount.dwarfs DEPENDS dwarfs-bin) list(APPEND SYMLINKS mount.dwarfs) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/mount.dwarfs DESTINATION sbin) install(TARGETS dwarfs-bin RUNTIME DESTINATION sbin) - endif() + endif(WINFSP) list(APPEND BINARY_TARGETS dwarfs-bin) list(APPEND MAIN_TARGETS dwarfs_main) - endif() -endif() + endif(FUSE3_FOUND OR WINFSP) if(FUSE_FOUND AND (WITH_LEGACY_FUSE OR NOT FUSE3_FOUND)) add_library(dwarfs2_main src/dwarfs_main.cpp) @@ -638,19 +623,20 @@ if(FUSE_FOUND AND (WITH_LEGACY_FUSE OR NOT FUSE3_FOUND)) target_link_libraries(dwarfs2_main PkgConfig::FUSE) add_executable(dwarfs2-bin src/dwarfs.cpp) target_link_libraries(dwarfs2-bin dwarfs2_main) - if(NOT FUSE3_FOUND) - target_link_libraries(dwarfsuniversal dwarfs2_main) - endif() - set_target_properties(dwarfs2-bin PROPERTIES OUTPUT_NAME dwarfs2) - add_custom_command(OUTPUT mount.dwarfs2 - COMMAND ${CMAKE_COMMAND} -E create_symlink dwarfs2 mount.dwarfs2 - DEPENDS dwarfs2-bin) - list(APPEND SYMLINKS mount.dwarfs2) - install(TARGETS dwarfs2-bin RUNTIME DESTINATION sbin) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/mount.dwarfs2 DESTINATION sbin) - list(APPEND BINARY_TARGETS dwarfs2-bin) - list(APPEND MAIN_TARGETS dwarfs2_main) -endif() + if(NOT FUSE3_FOUND) + target_link_libraries(dwarfsuniversal dwarfs2_main) + endif(NOT FUSE3_FOUND) + set_target_properties(dwarfs2-bin PROPERTIES OUTPUT_NAME dwarfs2) + add_custom_command(OUTPUT mount.dwarfs2 + COMMAND ${CMAKE_COMMAND} -E create_symlink dwarfs2 mount.dwarfs2 + DEPENDS dwarfs2-bin) + list(APPEND SYMLINKS mount.dwarfs2) + install(TARGETS dwarfs2-bin RUNTIME DESTINATION sbin) + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/mount.dwarfs2 DESTINATION sbin) + list(APPEND BINARY_TARGETS dwarfs2-bin) + list(APPEND MAIN_TARGETS dwarfs2_main) + endif(FUSE_FOUND AND (WITH_LEGACY_FUSE OR NOT FUSE3_FOUND)) +endif("${TEBAKO_BUILD_SCOPE}" STREQUAL "ALL") add_custom_target(symlinks ALL DEPENDS ${SYMLINKS}) diff --git a/include/dwarfs/console_writer.h b/include/dwarfs/console_writer.h index 3c789880a..40c04c6b6 100644 --- a/include/dwarfs/console_writer.h +++ b/include/dwarfs/console_writer.h @@ -35,7 +35,7 @@ namespace dwarfs { class progress; -class console_writer : public logger { +class console_writer : public stream_logger { public: using get_term_width_type = std::function; @@ -46,25 +46,20 @@ class console_writer : public logger { get_term_width_type get_term_width, level_type threshold, display_mode mode = NORMAL, bool verbose = false); - void write(level_type level, const std::string& output, char const* file, - int line) override; - void update(const progress& p, bool last); private: + void preamble() override; + void postamble() override; + std::string_view get_newline() const override; void rewind(); - std::ostream& os_; - std::mutex mx_; - std::atomic threshold_; std::string statebuf_; double frac_; std::atomic counter_{0}; progress_mode const pg_mode_; get_term_width_type get_term_width_; display_mode const mode_; - bool const color_; - bool const with_context_; bool const debug_progress_; bool writing_{false}; speedometer read_speed_; diff --git a/include/dwarfs/entry.h b/include/dwarfs/entry.h index 8e838115e..77166eece 100644 --- a/include/dwarfs/entry.h +++ b/include/dwarfs/entry.h @@ -101,12 +101,12 @@ class entry : public entry_interface { virtual std::optional const& inode_num() const = 0; // more methods from entry_interface - uint16_t get_permissions() const override; - void set_permissions(uint16_t perm) override; - uint16_t get_uid() const override; - void set_uid(uint16_t uid) override; - uint16_t get_gid() const override; - void set_gid(uint16_t gid) override; + mode_type get_permissions() const override; + void set_permissions(mode_type perm) override; + uid_type get_uid() const override; + void set_uid(uid_type uid) override; + gid_type get_gid() const override; + void set_gid(gid_type gid) override; uint64_t get_atime() const override; void set_atime(uint64_t atime) override; uint64_t get_mtime() const override; diff --git a/include/dwarfs/entry_interface.h b/include/dwarfs/entry_interface.h index 399fefc7e..bd53b890e 100644 --- a/include/dwarfs/entry_interface.h +++ b/include/dwarfs/entry_interface.h @@ -23,12 +23,17 @@ #include +#include "dwarfs/file_stat.h" #include "dwarfs/object.h" namespace dwarfs { class entry_interface : public object { public: + using uid_type = file_stat::uid_type; + using gid_type = file_stat::gid_type; + using mode_type = file_stat::mode_type; + virtual std::string path_as_string() const = 0; virtual std::string dpath() const = 0; virtual std::string unix_dpath() const = 0; @@ -37,12 +42,12 @@ class entry_interface : public object { virtual size_t size() const = 0; virtual bool is_directory() const = 0; - virtual uint16_t get_permissions() const = 0; - virtual void set_permissions(uint16_t perm) = 0; - virtual uint16_t get_uid() const = 0; - virtual void set_uid(uint16_t uid) = 0; - virtual uint16_t get_gid() const = 0; - virtual void set_gid(uint16_t gid) = 0; + virtual mode_type get_permissions() const = 0; + virtual void set_permissions(mode_type perm) = 0; + virtual uid_type get_uid() const = 0; + virtual void set_uid(uid_type uid) = 0; + virtual gid_type get_gid() const = 0; + virtual void set_gid(gid_type gid) = 0; virtual uint64_t get_atime() const = 0; virtual void set_atime(uint64_t atime) = 0; virtual uint64_t get_mtime() const = 0; diff --git a/include/dwarfs/global_entry_data.h b/include/dwarfs/global_entry_data.h index d672e9ff8..3697e27a2 100644 --- a/include/dwarfs/global_entry_data.h +++ b/include/dwarfs/global_entry_data.h @@ -28,21 +28,27 @@ #include +#include "dwarfs/file_stat.h" + namespace dwarfs { struct scanner_options; class global_entry_data { public: + using uid_type = file_stat::uid_type; + using gid_type = file_stat::gid_type; + using mode_type = file_stat::mode_type; + enum class timestamp_type { ATIME, MTIME, CTIME }; global_entry_data(scanner_options const& options) : options_(options) {} - void add_uid(uint16_t uid); - void add_gid(uint16_t gid); + void add_uid(uid_type uid); + void add_gid(gid_type gid); - void add_mode(uint16_t mode) { add(mode, modes_, next_mode_index_); } + void add_mode(mode_type mode) { add(mode, modes_, next_mode_index_); } void add_mtime(uint64_t time); void add_atime(uint64_t time); @@ -56,9 +62,9 @@ class global_entry_data { index(symlinks_); } - uint16_t get_uid_index(uint16_t uid) const; - uint16_t get_gid_index(uint16_t gid) const; - uint16_t get_mode_index(uint16_t mode) const; + size_t get_uid_index(uid_type uid) const; + size_t get_gid_index(gid_type gid) const; + size_t get_mode_index(mode_type mode) const; uint32_t get_name_index(std::string const& name) const; uint32_t get_symlink_table_entry(std::string const& link) const; @@ -67,9 +73,9 @@ class global_entry_data { uint64_t get_atime_offset(uint64_t time) const; uint64_t get_ctime_offset(uint64_t time) const; - std::vector get_uids() const; - std::vector get_gids() const; - std::vector get_modes() const; + std::vector get_uids() const; + std::vector get_gids() const; + std::vector get_modes() const; std::vector get_names() const; std::vector get_symlinks() const; @@ -85,8 +91,8 @@ class global_entry_data { static void index(map_type& map); - void - add(uint16_t val, map_type& map, uint16_t& next_index) { + template + void add(T val, map_type& map, T& next_index) { if (map.emplace(val, next_index).second) { ++next_index; } @@ -94,14 +100,14 @@ class global_entry_data { uint64_t get_time_offset(uint64_t time) const; - map_type uids_; - map_type gids_; - map_type modes_; + map_type uids_; + map_type gids_; + map_type modes_; map_type names_; map_type symlinks_; - uint16_t next_uid_index_{0}; - uint16_t next_gid_index_{0}; - uint16_t next_mode_index_{0}; + uid_type next_uid_index_{0}; + gid_type next_gid_index_{0}; + mode_type next_mode_index_{0}; uint64_t timestamp_base_{std::numeric_limits::max()}; scanner_options const& options_; }; diff --git a/include/dwarfs/logger.h b/include/dwarfs/logger.h index af50a289c..ca0e42948 100644 --- a/include/dwarfs/logger.h +++ b/include/dwarfs/logger.h @@ -88,11 +88,22 @@ class stream_logger : public logger { void set_threshold(level_type threshold); void set_with_context(bool with_context) { with_context_ = with_context; } + protected: + virtual void preamble(); + virtual void postamble(); + virtual std::string_view get_newline() const; + + std::ostream& log_stream() const { return os_; } + std::mutex& log_mutex() const { return mx_; } + bool log_is_colored() const { return color_; } + level_type log_threshold() const { return threshold_.load(); } + private: std::ostream& os_; - std::mutex mx_; + std::mutex mutable mx_; std::atomic threshold_; bool const color_; + bool const enable_stack_trace_; bool with_context_; }; @@ -306,7 +317,8 @@ class log_proxy { }; #define LOG_DETAIL_LEVEL(level, lgr, method) \ - if (lgr.is_enabled_for(::dwarfs::logger::level)) \ + if constexpr (std::decay_t::is_enabled_for( \ + ::dwarfs::logger::level)) \ lgr.method(__FILE__, __LINE__) #define LOG_PROXY(policy, lgr) ::dwarfs::log_proxy log_(lgr) diff --git a/include/dwarfs/metadata_types.h b/include/dwarfs/metadata_types.h index 3c791e3c1..3ec232e76 100644 --- a/include/dwarfs/metadata_types.h +++ b/include/dwarfs/metadata_types.h @@ -33,6 +33,7 @@ #include +#include "dwarfs/file_stat.h" #include "dwarfs/file_type.h" #include "dwarfs/string_table.h" @@ -90,15 +91,19 @@ class inode_view friend class dir_entry_view; public: - uint16_t mode() const; + using uid_type = file_stat::uid_type; + using gid_type = file_stat::gid_type; + using mode_type = file_stat::mode_type; + + mode_type mode() const; posix_file_type::value type() const { return posix_file_type::from_mode(mode()); } bool is_regular_file() const { return type() == posix_file_type::regular; } bool is_directory() const { return type() == posix_file_type::directory; } bool is_symlink() const { return type() == posix_file_type::symlink; } - uint16_t getuid() const; - uint16_t getgid() const; + uid_type getuid() const; + gid_type getgid() const; uint32_t inode_num() const { return inode_num_; } private: diff --git a/include/dwarfs/options.h b/include/dwarfs/options.h index dfc2e590d..c1edabc13 100644 --- a/include/dwarfs/options.h +++ b/include/dwarfs/options.h @@ -27,6 +27,7 @@ #include #include +#include "dwarfs/file_stat.h" #include "dwarfs/types.h" namespace dwarfs { @@ -97,8 +98,8 @@ struct file_order_options { struct scanner_options { file_order_options file_order; std::optional file_hash_algorithm{"xxh3-128"}; - std::optional uid; - std::optional gid; + std::optional uid; + std::optional gid; std::optional timestamp; bool keep_all_times{false}; bool remove_empty_dirs{false}; diff --git a/include/dwarfs/util.h b/include/dwarfs/util.h index 4c57807e6..5dc9ec9e5 100644 --- a/include/dwarfs/util.h +++ b/include/dwarfs/util.h @@ -60,4 +60,6 @@ void shorten_path_string(std::string& path, char separator, size_t max_len); std::filesystem::path canonical_path(std::filesystem::path p); +bool getenv_is_enabled(char const* var); + } // namespace dwarfs diff --git a/src/dwarfs/console_writer.cpp b/src/dwarfs/console_writer.cpp index 46de5f271..594101393 100644 --- a/src/dwarfs/console_writer.cpp +++ b/src/dwarfs/console_writer.cpp @@ -19,12 +19,9 @@ * along with dwarfs. If not, see . */ -#include #include #include -#include - #include #include "dwarfs/console_writer.h" @@ -46,15 +43,6 @@ constexpr std::array asc_bar{ constexpr std::array uni_bar{ {"▏", "▎", "▍", "▌", "▋", "▊", "▉", "█"}}; -bool is_debug_progress() { - if (auto var = ::getenv("DWARFS_DEBUG_PROGRESS")) { - if (auto val = folly::tryTo(var)) { - return *val; - } - } - return false; -} - std::string progress_bar(size_t width, double frac, bool unicode) { size_t barlen = 8 * width * frac; size_t w = barlen / 8; @@ -82,22 +70,13 @@ console_writer::console_writer(std::ostream& os, progress_mode pg_mode, get_term_width_type get_term_width, level_type threshold, display_mode mode, bool with_context) - : os_(os) - , threshold_(threshold) + : stream_logger(os, threshold, with_context) , frac_(0.0) , pg_mode_(pg_mode) , get_term_width_(get_term_width) , mode_(mode) - , color_(stream_is_fancy_terminal(os)) - , with_context_(with_context) - , debug_progress_(is_debug_progress()) - , read_speed_{std::chrono::seconds(5)} { - if (threshold > level_type::INFO) { - set_policy(); - } else { - set_policy(); - } -} + , debug_progress_(getenv_is_enabled("DWARFS_DEBUG_PROGRESS")) + , read_speed_{std::chrono::seconds(5)} {} void console_writer::rewind() { if (!statebuf_.empty()) { @@ -112,74 +91,34 @@ void console_writer::rewind() { break; } - os_ << '\r'; + auto& os = log_stream(); + + os << '\r'; for (int i = 0; i < lines; ++i) { - os_ << "\x1b[A"; + os << "\x1b[A"; } } } -void console_writer::write(level_type level, const std::string& output, - char const* file, int line) { - if (level <= threshold_) { - auto t = get_current_time_string(); - const char* prefix = ""; - const char* suffix = ""; - - if (color_) { - switch (level) { - case ERROR: - prefix = terminal_color(termcolor::BOLD_RED); - suffix = terminal_color(termcolor::NORMAL); - break; - - case WARN: - prefix = terminal_color(termcolor::BOLD_YELLOW); - suffix = terminal_color(termcolor::NORMAL); - break; - - default: - break; - } - } - - char lchar = logger::level_char(level); - std::string context; - - if (with_context_ && file) { - context = get_logger_context(file, line); - if (color_) { - context = folly::to( - suffix, terminal_color(termcolor::MAGENTA), context, - terminal_color(termcolor::NORMAL), prefix); - } - } - - std::lock_guard lock(mx_); +void console_writer::preamble() { rewind(); } - switch (pg_mode_) { - case UNICODE: - case ASCII: - rewind(); - os_ << prefix << lchar << ' ' << t << ' ' << context << output << suffix - << "\x1b[K\n"; - os_ << statebuf_; - break; - - default: - os_ << lchar << ' ' << t << ' ' << context << output << "\n"; - break; - } +void console_writer::postamble() { + if (pg_mode_ == UNICODE || pg_mode_ == ASCII) { + log_stream() << statebuf_; } } +std::string_view console_writer::get_newline() const { + return pg_mode_ != NONE ? "\x1b[K\n" : "\n"; +} + void console_writer::update(const progress& p, bool last) { if (pg_mode_ == NONE && !last) { return; } - const char* newline = pg_mode_ != NONE ? "\x1b[K\n" : "\n"; + auto newline = get_newline(); std::ostringstream oss; lazy_value width(get_term_width_); @@ -209,7 +148,7 @@ void console_writer::update(const progress& p, bool last) { if (fancy) { oss << terminal_colored(p.status(width.get()), termcolor::BOLD_CYAN, - color_) + log_is_colored()) << newline; } @@ -271,9 +210,9 @@ void console_writer::update(const progress& p, bool last) { } if (pg_mode_ == NONE) { - if (INFO <= threshold_) { - std::lock_guard lock(mx_); - os_ << oss.str(); + if (INFO <= log_threshold()) { + std::lock_guard lock(log_mutex()); + log_stream() << oss.str(); } return; } @@ -301,12 +240,12 @@ void console_writer::update(const progress& p, bool last) { if (tmp != statebuf_) { auto t = get_current_time_string(); statebuf_ = tmp; - std::lock_guard lock(mx_); - os_ << "- " << t << statebuf_ << "\n"; + std::lock_guard lock(log_mutex()); + log_stream() << "- " << t << statebuf_ << "\n"; } if (last) { - std::lock_guard lock(mx_); - os_ << oss.str(); + std::lock_guard lock(log_mutex()); + log_stream() << oss.str(); } } else { oss << progress_bar(width.get() - 6, frac_, pg_mode_ == UNICODE) @@ -315,13 +254,13 @@ void console_writer::update(const progress& p, bool last) { ++counter_; - std::lock_guard lock(mx_); + std::lock_guard lock(log_mutex()); rewind(); statebuf_ = oss.str(); - os_ << statebuf_; + log_stream() << statebuf_; } } diff --git a/src/dwarfs/entry.cpp b/src/dwarfs/entry.cpp index be97bd641..48741ef56 100644 --- a/src/dwarfs/entry.cpp +++ b/src/dwarfs/entry.cpp @@ -174,17 +174,17 @@ void entry::pack(thrift::metadata::inode_data& entry_v2, entry::type_t file::type() const { return E_FILE; } -uint16_t entry::get_permissions() const { return stat_.permissions(); } +auto entry::get_permissions() const -> mode_type { return stat_.permissions(); } -void entry::set_permissions(uint16_t perm) { stat_.set_permissions(perm); } +void entry::set_permissions(mode_type perm) { stat_.set_permissions(perm); } -uint16_t entry::get_uid() const { return stat_.uid; } +auto entry::get_uid() const -> uid_type { return stat_.uid; } -void entry::set_uid(uint16_t uid) { stat_.uid = uid; } +void entry::set_uid(uid_type uid) { stat_.uid = uid; } -uint16_t entry::get_gid() const { return stat_.gid; } +auto entry::get_gid() const -> gid_type { return stat_.gid; } -void entry::set_gid(uint16_t gid) { stat_.gid = gid; } +void entry::set_gid(gid_type gid) { stat_.gid = gid; } uint64_t entry::get_atime() const { return stat_.atime; } diff --git a/src/dwarfs/global_entry_data.cpp b/src/dwarfs/global_entry_data.cpp index e6a203048..1f762a0bd 100644 --- a/src/dwarfs/global_entry_data.cpp +++ b/src/dwarfs/global_entry_data.cpp @@ -35,23 +35,23 @@ std::vector global_entry_data::get_vector(map_type const& map) const { get<0>() | as(); } -std::vector global_entry_data::get_uids() const { +auto global_entry_data::get_uids() const -> std::vector { return get_vector(uids_); } -std::vector global_entry_data::get_gids() const { +auto global_entry_data::get_gids() const -> std::vector { return get_vector(gids_); } -std::vector global_entry_data::get_modes() const { +auto global_entry_data::get_modes() const -> std::vector { return get_vector(modes_); } -std::vector global_entry_data::get_names() const { +auto global_entry_data::get_names() const -> std::vector { return get_vector(names_); } -std::vector global_entry_data::get_symlinks() const { +auto global_entry_data::get_symlinks() const -> std::vector { return get_vector(symlinks_); } @@ -84,15 +84,15 @@ uint64_t global_entry_data::get_timestamp_base() const { options_.time_resolution_sec; } -uint16_t global_entry_data::get_uid_index(uint16_t uid) const { +size_t global_entry_data::get_uid_index(uid_type uid) const { return options_.uid ? *options_.uid : DWARFS_NOTHROW(uids_.at(uid)); } -uint16_t global_entry_data::get_gid_index(uint16_t gid) const { +size_t global_entry_data::get_gid_index(gid_type gid) const { return options_.gid ? *options_.gid : DWARFS_NOTHROW(gids_.at(gid)); } -uint16_t global_entry_data::get_mode_index(uint16_t mode) const { +size_t global_entry_data::get_mode_index(mode_type mode) const { return DWARFS_NOTHROW(modes_.at(mode)); } @@ -105,13 +105,13 @@ global_entry_data::get_symlink_table_entry(std::string const& link) const { return DWARFS_NOTHROW(symlinks_.at(link)); } -void global_entry_data::add_uid(uint16_t uid) { +void global_entry_data::add_uid(uid_type uid) { if (!options_.uid) { add(uid, uids_, next_uid_index_); } } -void global_entry_data::add_gid(uint16_t gid) { +void global_entry_data::add_gid(gid_type gid) { if (!options_.gid) { add(gid, gids_, next_gid_index_); } diff --git a/src/dwarfs/logger.cpp b/src/dwarfs/logger.cpp index 23ed7899f..2270a8f53 100644 --- a/src/dwarfs/logger.cpp +++ b/src/dwarfs/logger.cpp @@ -19,12 +19,13 @@ * along with dwarfs. If not, see . */ -#include #include #include #include #include +#include +#include #ifndef NDEBUG #include @@ -38,6 +39,7 @@ #include "dwarfs/logger.h" #include "dwarfs/terminal.h" +#include "dwarfs/util.h" namespace dwarfs { @@ -64,16 +66,22 @@ stream_logger::stream_logger(std::ostream& os, level_type threshold, bool with_context) : os_(os) , color_(stream_is_fancy_terminal(os)) + , enable_stack_trace_{getenv_is_enabled("DWARFS_LOGGER_STACK_TRACE")} , with_context_(with_context) { set_threshold(threshold); } +void stream_logger::preamble() {} +void stream_logger::postamble() {} +std::string_view stream_logger::get_newline() const { return "\n"; } + void stream_logger::write(level_type level, const std::string& output, char const* file, int line) { if (level <= threshold_) { auto t = get_current_time_string(); const char* prefix = ""; const char* suffix = ""; + auto newline = get_newline(); if (color_) { switch (level) { @@ -93,24 +101,33 @@ void stream_logger::write(level_type level, const std::string& output, } #if DWARFS_SYMBOLIZE - folly::symbolizer::StringSymbolizePrinter printer( - color_ ? folly::symbolizer::SymbolizePrinter::COLOR : 0); + std::string stacktrace; + std::vector st_lines; - if (threshold_ == TRACE) { + if (enable_stack_trace_) { using namespace folly::symbolizer; Symbolizer symbolizer(LocationInfoMode::FULL); - FrameArray<5> addresses; + FrameArray<8> addresses; getStackTraceSafe(addresses); symbolizer.symbolize(addresses); - printer.println(addresses, 0); + folly::symbolizer::StringSymbolizePrinter printer( + color_ ? folly::symbolizer::SymbolizePrinter::COLOR : 0); + printer.println(addresses, 3); + stacktrace = printer.str(); + folly::split('\n', stacktrace, st_lines); + if (st_lines.back().empty()) { + st_lines.pop_back(); + } } #endif char lchar = logger::level_char(level); std::string context; + size_t context_len = 0; if (with_context_ && file) { context = get_logger_context(file, line); + context_len = context.size(); if (color_) { context = folly::to( suffix, terminal_color(termcolor::MAGENTA), context, @@ -118,22 +135,44 @@ void stream_logger::write(level_type level, const std::string& output, } } + folly::small_vector lines; + folly::split('\n', output, lines); + + if (lines.back().empty()) { + lines.pop_back(); + } + + bool clear_ctx = true; + std::lock_guard lock(mx_); - os_ << prefix << lchar << ' ' << t << ' ' << context << output << suffix - << "\n"; + + preamble(); + + for (auto l : lines) { + os_ << prefix << lchar << ' ' << t << ' ' << context << l << suffix + << newline; + + if (clear_ctx) { + std::fill(t.begin(), t.end(), '.'); + context.assign(context_len, ' '); + clear_ctx = false; + } + } #if DWARFS_SYMBOLIZE - if (threshold_ == TRACE) { - os_ << printer.str(); + for (auto l : st_lines) { + os_ << l << newline; } #endif + + postamble(); } } void stream_logger::set_threshold(level_type threshold) { threshold_ = threshold; - if (threshold > level_type::INFO) { + if (threshold >= level_type::DEBUG) { set_policy(); } else { set_policy(); diff --git a/src/dwarfs/metadata_types.cpp b/src/dwarfs/metadata_types.cpp index c06f38064..d387422e7 100644 --- a/src/dwarfs/metadata_types.cpp +++ b/src/dwarfs/metadata_types.cpp @@ -534,11 +534,17 @@ uint32_t global_metadata::parent_dir_entry(uint32_t ino) const { : meta_->directories()[ino].parent_entry(); } -uint16_t inode_view::mode() const { return meta_->modes()[mode_index()]; } +auto inode_view::mode() const -> mode_type { + return meta_->modes()[mode_index()]; +} -uint16_t inode_view::getuid() const { return meta_->uids()[owner_index()]; } +auto inode_view::getuid() const -> uid_type { + return meta_->uids()[owner_index()]; +} -uint16_t inode_view::getgid() const { return meta_->gids()[group_index()]; } +auto inode_view::getgid() const -> gid_type { + return meta_->gids()[group_index()]; +} // TODO: pretty certain some of this stuff can be simplified diff --git a/src/dwarfs/util.cpp b/src/dwarfs/util.cpp index 98ef7dde8..45df1d475 100644 --- a/src/dwarfs/util.cpp +++ b/src/dwarfs/util.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -187,4 +188,13 @@ std::filesystem::path canonical_path(std::filesystem::path p) { return p; } +bool getenv_is_enabled(char const* var) { + if (auto val = std::getenv(var)) { + if (auto maybeBool = folly::tryTo(val); maybeBool && *maybeBool) { + return true; + } + } + return false; +} + } // namespace dwarfs diff --git a/test/dwarfs.cpp b/test/dwarfs.cpp index f51e85b33..ce82df8e5 100644 --- a/test/dwarfs.cpp +++ b/test/dwarfs.cpp @@ -50,6 +50,7 @@ #include "loremipsum.h" #include "mmap_mock.h" #include "test_helpers.h" +#include "test_logger.h" using namespace dwarfs; @@ -134,9 +135,7 @@ void basic_end_to_end_test(std::string const& compressor, options.timestamp = 4711; } - std::ostringstream logss; - stream_logger lgr(logss); // TODO: mock - lgr.set_policy(); + test::test_logger lgr; auto input = test::os_access_mock::create_test_instance(); @@ -557,9 +556,7 @@ TEST_P(packing_test, regression_empty_fs) { options.pack_symlinks_index = pack_symlinks_index; options.force_pack_string_tables = true; - std::ostringstream logss; - stream_logger lgr(logss); // TODO: mock - lgr.set_policy(); + test::test_logger lgr; auto input = std::make_shared(); @@ -632,9 +629,7 @@ TEST(block_manager, regression_block_boundary) { opts.block_cache.max_bytes = 1 << 20; opts.metadata.check_consistency = true; - std::ostringstream logss; - stream_logger lgr(logss); // TODO: mock - lgr.set_policy(); + test::test_logger lgr; std::vector fs_blocks; @@ -681,9 +676,7 @@ TEST_P(compression_regression, github45) { opts.block_cache.max_bytes = 1 << 20; opts.metadata.check_consistency = true; - std::ostringstream logss; - stream_logger lgr(logss); // TODO: mock - lgr.set_policy(); + test::test_logger lgr; std::independent_bits_engine::digits, uint16_t> @@ -762,9 +755,7 @@ class file_scanner TEST_P(file_scanner, inode_ordering) { auto [order_mode, file_hash_algo] = GetParam(); - std::ostringstream logss; - stream_logger lgr(logss); // TODO: mock - lgr.set_policy(); + test::test_logger lgr; auto bmcfg = block_manager::config(); auto opts = scanner_options(); @@ -813,9 +804,7 @@ TEST_P(filter, filesystem) { options.remove_empty_dirs = true; - std::ostringstream logss; - stream_logger lgr(logss); // TODO: mock - lgr.set_policy(); + test::test_logger lgr; auto scr = std::make_shared(lgr); @@ -866,9 +855,7 @@ INSTANTIATE_TEST_SUITE_P(dwarfs, filter, ::testing::ValuesIn(dwarfs::test::get_filter_tests())); TEST(file_scanner, input_list) { - std::ostringstream logss; - stream_logger lgr(logss); // TODO: mock - lgr.set_policy(); + test::test_logger lgr; auto bmcfg = block_manager::config(); auto opts = scanner_options(); @@ -902,3 +889,35 @@ TEST(file_scanner, input_list) { EXPECT_EQ(expected, got); } + +TEST(filesystem, uid_gid_32bit) { + test::test_logger lgr; + + auto input = std::make_shared(); + + input->add("", {1, 040755, 1, 0, 0, 10, 42, 0, 0, 0}); + input->add("foo16.txt", {2, 0100755, 1, 60000, 65535, 5, 42, 0, 0, 0}, "hello"); + input->add("foo32.txt", {3, 0100755, 1, 65536, 4294967295, 5, 42, 0, 0, 0}, "world"); + + auto fsimage = build_dwarfs(lgr, input, "null"); + + auto mm = std::make_shared(std::move(fsimage)); + + filesystem_v2 fs(lgr, mm); + + auto iv16 = fs.find("/foo16.txt"); + auto iv32 = fs.find("/foo32.txt"); + + EXPECT_TRUE(iv16); + EXPECT_TRUE(iv32); + + file_stat st16, st32; + + EXPECT_EQ(0, fs.getattr(*iv16, &st16)); + EXPECT_EQ(0, fs.getattr(*iv32, &st32)); + + EXPECT_EQ(60000, st16.uid); + EXPECT_EQ(65535, st16.gid); + EXPECT_EQ(65536, st32.uid); + EXPECT_EQ(4294967295, st32.gid); +} diff --git a/test/dwarfs_badfs.cpp b/test/dwarfs_badfs.cpp index 6a7748cad..cffafa5d1 100644 --- a/test/dwarfs_badfs.cpp +++ b/test/dwarfs_badfs.cpp @@ -29,6 +29,8 @@ #include "dwarfs/mmap.h" #include "dwarfs/options.h" +#include "test_logger.h" + using namespace dwarfs; namespace { @@ -52,14 +54,14 @@ class bad_fs : public ::testing::TestWithParam {}; TEST_P(bad_fs, test) { auto filename = testdata / GetParam(); - std::ostringstream oss, oss2; - stream_logger lgr(oss); + test::test_logger lgr; + std::ostringstream oss; int nerror = 0; try { nerror = - filesystem_v2::identify(lgr, std::make_shared(filename), oss2, 9, + filesystem_v2::identify(lgr, std::make_shared(filename), oss, 9, 1, true, filesystem_options::IMAGE_OFFSET_AUTO); } catch (std::exception const&) { nerror = 1; diff --git a/test/dwarfs_compat.cpp b/test/dwarfs_compat.cpp index bb4b2b8ee..1db52499f 100644 --- a/test/dwarfs_compat.cpp +++ b/test/dwarfs_compat.cpp @@ -48,6 +48,7 @@ #include "dwarfs/worker_group.h" #include "mmap_mock.h" +#include "test_logger.h" using namespace dwarfs; @@ -1037,8 +1038,7 @@ void check_dynamic(std::string const& version, filesystem_v2 const& fs) { } TEST_P(compat_metadata, backwards_compat) { - std::ostringstream oss; - stream_logger lgr(oss); + test::test_logger lgr; auto version = GetParam(); auto filename = std::string(TEST_DATA_DIR "/compat-v") + version + ".dwarfs"; filesystem_v2 fs(lgr, std::make_shared(filename)); @@ -1054,8 +1054,7 @@ class compat_filesystem TEST_P(compat_filesystem, backwards_compat) { auto [version, enable_nlink] = GetParam(); - std::ostringstream oss; - stream_logger lgr(oss); + test::test_logger lgr; auto filename = std::string(TEST_DATA_DIR "/compat-v") + version + ".dwarfs"; filesystem_options opts; @@ -1097,8 +1096,7 @@ class rewrite TEST_P(rewrite, filesystem_rewrite) { auto [version, recompress_block, recompress_metadata] = GetParam(); - std::ostringstream oss; - stream_logger lgr(oss); + test::test_logger lgr; auto filename = std::string(TEST_DATA_DIR "/compat-v") + version + ".dwarfs"; rewrite_options opts; diff --git a/test/test_logger.h b/test/test_logger.h new file mode 100644 index 000000000..3fdac7f9e --- /dev/null +++ b/test/test_logger.h @@ -0,0 +1,102 @@ +/* vim:set ts=2 sw=2 sts=2 et: */ +/** + * \author Marcus Holland-Moritz (github@mhxnet.de) + * \copyright Copyright (c) Marcus Holland-Moritz + * + * This file is part of dwarfs. + * + * dwarfs is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * dwarfs is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with dwarfs. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "dwarfs/logger.h" +#include "dwarfs/util.h" + +namespace dwarfs::test { + +class test_logger : public ::dwarfs::logger { + public: + struct log_entry { + log_entry(level_type level, std::string const& output, char const* file, + int line) + : level{level} + , output{output} + , file{file} + , line{line} {} + + level_type level; + std::string output; + char const* file; + int line; + }; + + test_logger(std::optional threshold = std::nullopt) + : threshold_{threshold ? *threshold : default_threshold()} + , output_threshold_{output_threshold(default_threshold())} + , output_{::dwarfs::getenv_is_enabled("DWARFS_TEST_LOGGER_OUTPUT")} { + if (threshold_ >= level_type::DEBUG || + (output_ && output_threshold_ > level_type::DEBUG)) { + set_policy(); + } else { + set_policy(); + } + } + + void write(level_type level, std::string const& output, char const* file, + int line) override { + if (output_ && level <= output_threshold_) { + std::lock_guard lock(mx_); + std::cerr << level_char(level) << " [" << file << ":" << line << "] " + << output << "\n"; + } + + if (level <= threshold_) { + std::lock_guard lock(mx_); + log_.emplace_back(level, output, file, line); + } + } + + std::vector const& get_log() const { return log_; } + + bool empty() const { return log_.empty(); } + + void clear() { log_.clear(); } + + private: + static level_type default_threshold() { return level_type::INFO; } + + static level_type output_threshold(level_type default_level) { + if (auto var = std::getenv("DWARFS_TEST_LOGGER_LEVEL")) { + return ::dwarfs::logger::parse_level(var); + } + return default_level; + } + + std::mutex mx_; + std::vector log_; + level_type const threshold_; + level_type const output_threshold_; + bool const output_; +}; + +} // namespace dwarfs::test diff --git a/thrift/metadata.thrift b/thrift/metadata.thrift index 72eb7cd06..7ed3bf9a7 100644 --- a/thrift/metadata.thrift +++ b/thrift/metadata.thrift @@ -244,13 +244,13 @@ struct metadata { 6: list symlink_table // user ids, for lookup by `inode.owner_index` - 7: list uids + 7: list uids // group ids, for lookup by `inode.group_index` - 8: list gids + 8: list gids // inode modes, for lookup by `inode.mode_index` - 9: list modes + 9: list modes // directory entry names, for lookup by `dir_entry.name_index` 10: list names