From 2b27c9746c0568be09186877163bd3fb57477c86 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Wed, 8 Nov 2023 22:38:24 +0100 Subject: [PATCH 1/7] docs: fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 41cd9cce2..9bcae81c2 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ $ ln -s dwarfs-universal-0.7.0-Linux-aarch64 mkdwarfs $ ./mkdwarfs --help ``` -This also works on Windows ift the file system supports symbolic links: +This also works on Windows if the file system supports symbolic links: ``` > mklink mkdwarfs.exe dwarfs-universal-0.7.0-Windows-AMD64.exe From 79207af1ce0096cd4b075c3b7c8b0c8b7a715940 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Wed, 8 Nov 2023 22:38:46 +0100 Subject: [PATCH 2/7] docs: document current issues with WinFsp --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 9bcae81c2..bc830ef74 100644 --- a/README.md +++ b/README.md @@ -412,6 +412,12 @@ There are a few things worth pointing out, though: a DwarFS image that contains links of some sort, you might run into errors if you don't have the right privileges. +- Due to a [problem](https://github.com/winfsp/winfsp/issues/454) in + WinFsp, symlinks cannot currently point outside of the mounted file + system. Furthermore, due to another + [problem](https://github.com/winfsp/winfsp/issues/530) in WinFsp, + symlinks with a drive letter will appear with a mangled target path. + - The DwarFS driver on Windows correctly reports hardlink counts via its API, but currently these counts are not correctly propagated to the Windows file system layer. This is presumably due to a From 6fe969bcd1d1ade1ddb5d364d90db18e25efc031 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Sun, 19 Nov 2023 09:16:27 +0100 Subject: [PATCH 3/7] build: move docker to ubuntu 23.10 --- .docker/Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 3f7697498..2f9d5a892 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -1,16 +1,16 @@ # syntax=docker/dockerfile:1 ARG ARCH=amd64 -FROM $ARCH/ubuntu:22.04 +FROM $ARCH/ubuntu:23.10 RUN apt update && apt install -y gnupg wget software-properties-common RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ jammy main' RUN apt install -y \ - gcc \ - g++ \ gcc-12 \ g++-12 \ - clang \ - clang-15 \ + gcc-13 \ + g++-13 \ + clang-16 \ + clang-17 \ git \ vim \ zstd \ @@ -52,13 +52,14 @@ RUN apt install -y \ libfuse3-dev \ libgoogle-glog-dev \ libutfcpp-dev \ + libflac++-dev \ bash-completion \ upx COPY install-static-libs.sh /usr/local/bin/install-static-libs.sh RUN bash /usr/local/bin/install-static-libs.sh -RUN useradd -g users -u 1000 -m mhx +# RUN useradd -g users -u 1000 -m mhx ARG SCRIPT=build-linux.sh COPY $SCRIPT /usr/local/bin/run.sh RUN chmod +x /usr/local/bin/run.sh -USER mhx +USER ubuntu ENTRYPOINT /usr/local/bin/run.sh From 86b78b0c759c4094be7ea0013a2d0d69c0d4de7a Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Sun, 19 Nov 2023 09:31:14 +0100 Subject: [PATCH 4/7] build: update build script / workflow --- .docker/build-linux.sh | 14 +++++++------- .docker/install-static-libs.sh | 12 ++++++------ .github/workflows/build.yml | 19 ++++++++++--------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/.docker/build-linux.sh b/.docker/build-linux.sh index c82dfce85..4063124ac 100644 --- a/.docker/build-linux.sh +++ b/.docker/build-linux.sh @@ -29,16 +29,16 @@ esac case "-$BUILD_TYPE-" in *-gcc-*) - export CC=gcc CXX=g++ + export CC=gcc-13 CXX=g++-13 ;; - *-clang-*) - export CC=clang CXX=clang++ - ;; - *-gcc12-*) + *-oldgcc-*) export CC=gcc-12 CXX=g++-12 ;; - *-clang15-*) - export CC=clang-15 CXX=clang++-15 + *-clang-*) + export CC=clang-17 CXX=clang++-17 + ;; + *-oldclang-*) + export CC=clang-16 CXX=clang++-16 ;; *) echo "missing compiler in: $BUILD_TYPE" diff --git a/.docker/install-static-libs.sh b/.docker/install-static-libs.sh index cddabf016..d85e6c020 100644 --- a/.docker/install-static-libs.sh +++ b/.docker/install-static-libs.sh @@ -6,18 +6,18 @@ cd "$HOME" mkdir pkgs cd pkgs -wget https://github.com/libarchive/libarchive/releases/download/v3.6.2/libarchive-3.6.2.tar.xz -wget ftp://ftp.astron.com/pub/file/file-5.44.tar.gz +wget https://github.com/libarchive/libarchive/releases/download/v3.7.2/libarchive-3.7.2.tar.xz +wget ftp://ftp.astron.com/pub/file/file-5.45.tar.gz -tar xf libarchive-3.6.2.tar.xz -cd libarchive-3.6.2 +tar xf libarchive-3.7.2.tar.xz +cd libarchive-3.7.2 ./configure --prefix=/opt/static-libs --without-iconv --without-xml2 --without-expat make -j$(nproc) make install cd "$HOME/pkgs" -tar xf file-5.44.tar.gz -cd file-5.44 +tar xf file-5.45.tar.gz +cd file-5.45 ./configure --prefix=/opt/static-libs --enable-static=yes --enable-shared=no make -j$(nproc) make install diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dd2a25a9c..93ac7a727 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,20 +76,21 @@ jobs: - amd64 - arm64v8 build_type: - - gcc-debug-make - - gcc12-debug-ninja - - gcc12-release-ninja - - clang15-release-ninja - - clang15-release-ninja-static + - gcc-debug-ninja + - gcc-release-ninja + - oldgcc-debug-make + - clang-release-ninja + - clang-release-ninja-static + - oldclang-debug-make include: - arch: amd64 - build_type: clang15-asan-ninja + build_type: clang-asan-ninja - arch: amd64 - build_type: clang15-tsan-ninja + build_type: clang-tsan-ninja - arch: amd64 - build_type: clang15-ubsan-ninja + build_type: clang-ubsan-ninja - arch: amd64 - build_type: gcc12-debug-noperfmon-ninja + build_type: gcc-debug-noperfmon-ninja runs-on: - self-hosted From 754eb33bb807dcdd5975fc7518ccf79070d74475 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Mon, 20 Nov 2023 11:05:25 +0100 Subject: [PATCH 5/7] chore: remove python support leftovers --- scripts/example.py | 78 ---------------------------------------------- 1 file changed, 78 deletions(-) delete mode 100644 scripts/example.py diff --git a/scripts/example.py b/scripts/example.py deleted file mode 100644 index 93d724fdc..000000000 --- a/scripts/example.py +++ /dev/null @@ -1,78 +0,0 @@ -class mkdwarfs(object): - """ - The class defining mkdwarfs customization. - - If this is named `mkdwarfs`, you only have to specify the path to - the script file with `--script`. You can define multiple classes in - a single script, in which case you'll have to pass the class name - in addition to the script path as `--script :`. If the - class has a custom contructor, it is also possible to pass arguments - to the constuctor from the command line. - - All methods are optional. If you want to define methods beyond the - ones specified below, make sure you start their names with an - underscore, otherwise there will be a warning to ensure you don't - accidentally mistype the names of the methods. - - You can use the global `logger` object for logging. - """ - def __init__(self): - """ - Optional constructor - """ - logger.info("this is python!") - - def configure(self, config): - """ - Configuration - - This will be called early and allows you to change the default - for or even override (command line) parameters. Only a small - number of parameters are currently supported. - """ - # Enable similarity hash computation, useful if you actually - # want to use it in the `order` method. - config.enable_similarity() - config.set_order(file_order_mode.script, set_mode.override) - config.set_remove_empty_dirs(True, set_mode.default) - - def filter(self, entry): - """ - Filtering - - This will be called for every file system entry. If you return - `False`, the entry will be skipped. - """ - logger.debug(f"filter: {entry.path()} [{entry.type()}]") - if entry.type() == 'directory' and entry.name() == 'dev': - return False - return True - - def transform(self, entry): - """ - Transformation - - This will be called for every entry that has not been filtered, - and allows you to change certain attributes, such as permissions, - ownership, or timestamps. - """ - logger.debug(f"transform {entry.path()}") - entry.set_permissions(entry.permissions() & 0o7555) - return entry - - def order(self, inodes): - """ - Inode Ordering - - This will be called for every regular file inode, after all - entries have been scanned and files have been deduplicated. - """ - logger.info("order") - for i in inodes: - logger.debug(f"inode: {i.similarity_hash()} {i.size()} {i.refcount()}") - for p in i.paths(): - logger.debug(f" file: {p}") - return reversed(inodes) - - def _something_private(self): - pass From 99e76800900d3e487957e1692f7a3d078b1fe23b Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Mon, 20 Nov 2023 22:54:29 +0100 Subject: [PATCH 6/7] chore: fix thrift deprecation warnings --- CMakeLists.txt | 2 ++ thrift/metadata.thrift | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index febd1efed..9b400a262 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -624,6 +624,7 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift/thrift/lib/thrift/frozen.thrift ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift/ COMMAND ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 + -I ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift -o ${CMAKE_CURRENT_BINARY_DIR}/thrift/lib/thrift --gen mstch_cpp2 frozen.thrift @@ -645,6 +646,7 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/thrift/metadata.thrift ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs/metadata.thrift COMMAND ${CMAKE_CURRENT_BINARY_DIR}/bin/thrift1 + -I ${CMAKE_CURRENT_SOURCE_DIR}/fbthrift -o ${CMAKE_CURRENT_BINARY_DIR}/thrift/dwarfs --gen mstch_cpp2:frozen2 metadata.thrift diff --git a/thrift/metadata.thrift b/thrift/metadata.thrift index bb9a5b7ce..0872bcbc1 100644 --- a/thrift/metadata.thrift +++ b/thrift/metadata.thrift @@ -19,11 +19,16 @@ * along with dwarfs. If not, see . */ +include "thrift/annotation/cpp.thrift" + namespace cpp2 dwarfs.thrift.metadata -typedef i16 (cpp2.type = "uint16_t") UInt16 -typedef i32 (cpp2.type = "uint32_t") UInt32 -typedef i64 (cpp2.type = "uint64_t") UInt64 +@cpp.Type{name = "uint16_t"} +typedef i16 UInt16 +@cpp.Type{name = "uint32_t"} +typedef i32 UInt32 +@cpp.Type{name = "uint64_t"} +typedef i64 UInt64 /** * One chunk of data From 6214ccbe8c0c615224087bf97c9b700c590acab7 Mon Sep 17 00:00:00 2001 From: Marcus Holland-Moritz Date: Mon, 20 Nov 2023 22:54:40 +0100 Subject: [PATCH 7/7] chore: update folly/fbthrift --- fbthrift | 2 +- folly | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fbthrift b/fbthrift index 19890bdbf..0d87d2068 160000 --- a/fbthrift +++ b/fbthrift @@ -1 +1 @@ -Subproject commit 19890bdbf8ace9df6364162f1d7284b8ecb9df4e +Subproject commit 0d87d2068b9df8ae5b7ab21048ffeb58ba91a9ca diff --git a/folly b/folly index d2e577a8c..3a0cd44fc 160000 --- a/folly +++ b/folly @@ -1 +1 @@ -Subproject commit d2e577a8c8911d6f394d4cba4467831889efbfb7 +Subproject commit 3a0cd44fc76e1453cc2565cbd4d2a41caf355eb4