Update from upstream

.codecov.yml

@@ -0,0 +1,2 @@
+ignore:
+  - "fsst/"

.docker/Dockerfile

@@ -37,6 +37,7 @@ RUN apt install -y \
         libboost-thread-dev \
         libbrotli-dev \
         libevent-dev \
+        libhowardhinnant-date-dev \
         libjemalloc-dev \
         libdouble-conversion-dev \
         libiberty-dev \
@@ -53,10 +54,15 @@ RUN apt install -y \
         libgoogle-glog-dev \
         libutfcpp-dev \
         libflac++-dev \
-        bash-completion \
-        upx
+        python3-mistletoe \
+        bash-completion
 COPY install-static-libs.sh /usr/local/bin/install-static-libs.sh
 RUN bash /usr/local/bin/install-static-libs.sh
+ARG ARCH
+# upx-4.2.2 is broken for arm64 (https://github.com/upx/upx/issues/758)
+RUN wget -O - https://github.com/upx/upx/releases/download/v4.2.1/upx-4.2.1-$(bash -c "echo \${0//v8/}" $ARCH)_linux.tar.xz | tar -xJf - -C /usr/local/bin --strip-components=1 --wildcards "*/upx"
+# current ronn version is horribly broken
+RUN gem install ronn-ng
 # RUN useradd -g users -u 1000 -m mhx
 ARG SCRIPT=build-linux.sh
 COPY $SCRIPT /usr/local/bin/run.sh

.docker/Makefile

@@ -1,15 +1,20 @@
 IMAGE="dwarfs-buildenv"
 
 ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+NPROC:=$(shell nproc)
 
 build:
 	docker build -t $(IMAGE) .
 
+build_arm:
+	docker build -t $(IMAGE) --build-arg ARCH=arm64v8 .
+
 run:
 	@docker run -it --rm \
             --cap-add SYS_ADMIN --device /dev/fuse --privileged \
             --mount type=bind,source="$(ROOT_DIR)"/..,target=/workspace,readonly \
             --mount type=bind,source="$(ROOT_DIR)"/../@docker-ccache,target=/ccache \
-            --mount type=bind,source="$(ROOT_DIR)"/../@docker-build,target=/home/mhx/build \
-            --env CCACHE_DIR=/ccache --env CTEST_PARALLEL_LEVEL=$(nproc) \
+            --mount type=bind,source="$(ROOT_DIR)"/../@docker-home,target=/home/ubuntu \
+            --mount type=bind,source="$(ROOT_DIR)"/../@docker-build,target=/home/ubuntu/build \
+            --env CCACHE_DIR=/ccache --env CTEST_PARALLEL_LEVEL=$(NPROC) \
             --entrypoint /bin/bash $(IMAGE)

.docker/build-linux.sh

@@ -6,6 +6,8 @@ export CCACHE_DIR=/ccache
 
 cd "$HOME"
 
+git config --global --add safe.directory /workspace
+
 rm -f dwarfs
 ln -s /workspace dwarfs
 
@@ -30,12 +32,14 @@ esac
 case "-$BUILD_TYPE-" in
   *-gcc-*)
     export CC=gcc-13 CXX=g++-13
+    export COMPILER=gcc
     ;;
   *-oldgcc-*)
     export CC=gcc-12 CXX=g++-12
     ;;
   *-clang-*)
     export CC=clang-17 CXX=clang++-17
+    export COMPILER=clang
     ;;
   *-oldclang-*)
     export CC=clang-16 CXX=clang++-16
@@ -52,31 +56,42 @@ case "-$BUILD_TYPE-" in
   *-release-*)
     CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Release"
     ;;
+  *-reldbg-*)
+    CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo"
+    ;;
   *-asan-*)
     CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=1"
     ;;
   *-tsan-*)
     CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_TSAN=1"
+    export TSAN_OPTIONS="suppressions=/workspace/tsan.supp"
     ;;
   *-ubsan-*)
     CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_UBSAN=1"
     ;;
+  *-coverage-*)
+    CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_COVERAGE=1"
+    ;;
   *)
     echo "missing build type: $BUILD_TYPE"
     exit 1
 esac
 
+if [[ "-$BUILD_TYPE-" == *-O2-* ]]; then
+  CMAKE_ARGS="${CMAKE_ARGS} -DDWARFS_OPTIMIZE=2"
+fi
+
 if [[ "-$BUILD_TYPE-" == *-nojemalloc-* ]]; then
   CMAKE_ARGS="${CMAKE_ARGS} -DUSE_JEMALLOC=0"
 fi
 
 if [[ "-$BUILD_TYPE-" == *-noperfmon-* ]]; then
-  CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PERFMON=0"
+  CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PERFMON=0 -DWITH_MAN_OPTION=0"
 fi
 
 if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
   CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_DO_NOT_USE=1"
-  CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_EXTRA_PREFIX=/opt/static-libs"
+  CMAKE_ARGS="${CMAKE_ARGS} -DSTATIC_BUILD_EXTRA_PREFIX=/opt/static-libs/$COMPILER"
 else
   CMAKE_ARGS="${CMAKE_ARGS} -DWITH_BENCHMARKS=1"
 fi
@@ -89,47 +104,72 @@ cmake ../dwarfs/ $CMAKE_ARGS
 
 $BUILD_TOOL
 
+if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then
+  export LLVM_PROFILE_FILE="$PWD/profile/%32m.profraw"
+fi
+
 ctest --output-on-failure -j$(nproc)
 
-if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
-  $BUILD_TOOL package_source
-  if [[ "$BUILD_ARCH" == "amd64" ]]; then
-    $BUILD_TOOL copy_source_artifacts
-  fi
+if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then
+  unset LLVM_PROFILE_FILE
+  rm -rf /tmp-runner/coverage
+  mkdir -p /tmp-runner/coverage
+  llvm-profdata-17 merge -sparse profile/* -o dwarfs.profdata
+  for binary in mkdwarfs dwarfs dwarfsck dwarfsextract *_test; do
+    llvm-cov-17 show -instr-profile=dwarfs.profdata $binary >/tmp-runner/coverage/$binary.txt
+  done
 fi
 
-$BUILD_TOOL realclean
-
 if [[ "-$BUILD_TYPE-" == *-static-* ]]; then
-  cd "$HOME"
+  if [[ "-$BUILD_TYPE-" == *-release-* ]]; then
+    # in the clang-release-static case, we also try to build from the source tarball
+    if [[ "-$BUILD_TYPE-" == *-clang-* ]] && [[ "-$BUILD_TYPE-" != *-O2-* ]]; then
+      $BUILD_TOOL package_source
+
+      if [[ "$BUILD_ARCH" == "amd64" ]]; then
+        $BUILD_TOOL copy_source_artifacts
+      fi
+
+      $BUILD_TOOL realclean
+
+      cd "$HOME"
 
-  VERSION=$(git -C /workspace describe --tags --match "v*" --dirty)
-  VERSION=${VERSION:1}
+      VERSION=$(git -C /workspace describe --tags --match "v*" --dirty --abbrev=10)
+      VERSION=${VERSION:1}
 
-  rm -rf dwarfs-*
-  rm -f dwarfs
+      rm -rf dwarfs-*
+      rm -f dwarfs
 
-  mv "build/dwarfs-${VERSION}.tar.zst" .
-  rm -rf build
+      mv "build/dwarfs-${VERSION}.tar.zst" .
+      rm -rf build
 
-  tar xvf "dwarfs-${VERSION}.tar.zst"
-  ln -s "dwarfs-${VERSION}" dwarfs
+      tar xvf "dwarfs-${VERSION}.tar.zst"
+      mv "dwarfs-${VERSION}" dwarfs
 
-  mkdir build
-  cd build
+      mkdir build
+      cd build
 
-  # shellcheck disable=SC2086
-  cmake ../dwarfs/ $CMAKE_ARGS
+      # shellcheck disable=SC2086
+      cmake ../dwarfs/ $CMAKE_ARGS
 
-  $BUILD_TOOL
+      $BUILD_TOOL
 
-  ctest --output-on-failure -j$(nproc)
+      ctest --output-on-failure -j$(nproc)
+    fi
+
+    $BUILD_TOOL strip
+  fi
 
-  $BUILD_TOOL strip
   $BUILD_TOOL package
   $BUILD_TOOL universal_upx
 
   $BUILD_TOOL copy_artifacts
 
-  $BUILD_TOOL realclean
+  rm -rf /tmp-runner/artifacts
+  mkdir -p /tmp-runner/artifacts
+  cp artifacts.env /tmp-runner
+  cp dwarfs-universal-* /tmp-runner/artifacts
+  cp dwarfs-*-Linux*.tar.zst /tmp-runner/artifacts
 fi
+
+$BUILD_TOOL realclean

.docker/install-static-libs.sh

@@ -6,21 +6,86 @@ cd "$HOME"
 mkdir pkgs
 cd pkgs
 
-wget https://github.com/libarchive/libarchive/releases/download/v3.7.2/libarchive-3.7.2.tar.xz
-wget ftp://ftp.astron.com/pub/file/file-5.45.tar.gz
-
-tar xf libarchive-3.7.2.tar.xz
-cd libarchive-3.7.2
-./configure --prefix=/opt/static-libs --without-iconv --without-xml2 --without-expat
-make -j$(nproc)
-make install
-
-cd "$HOME/pkgs"
-tar xf file-5.45.tar.gz
-cd file-5.45
-./configure --prefix=/opt/static-libs --enable-static=yes --enable-shared=no
-make -j$(nproc)
-make install
+FILE_VERSION=5.45
+FILE_SHA512=12611a59ff766c22a55db4b4a9f80f95a0a2e916a1d8593612c6ead32c247102a8fdc23693c6bf81bda9b604d951a62c0051e91580b1b79e190a3504c0efc20a
+LIBARCHIVE_VERSION=3.7.2
+FLAC_VERSION=1.4.3
+LIBUNWIND_VERSION=1.7.2
+# BENCHMARK_VERSION=1.8.2
+
+RETRY=0
+while true; do
+    rm -f file-${FILE_VERSION}.tar.gz
+    wget ftp://ftp.astron.com/pub/file/file-${FILE_VERSION}.tar.gz
+    if echo "${FILE_SHA512}  file-${FILE_VERSION}.tar.gz" | sha512sum -c; then
+        break
+    fi
+    RETRY=$((RETRY+1))
+    if [ $RETRY -gt 10 ]; then
+        echo "Failed to download file-${FILE_VERSION}.tar.gz"
+        exit 1
+    fi
+done
+
+wget https://github.com/libarchive/libarchive/releases/download/v${LIBARCHIVE_VERSION}/libarchive-${LIBARCHIVE_VERSION}.tar.xz
+wget https://github.com/xiph/flac/releases/download/${FLAC_VERSION}/flac-${FLAC_VERSION}.tar.xz
+wget https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/libunwind-${LIBUNWIND_VERSION}.tar.gz
+# wget https://github.com/google/benchmark/archive/refs/tags/v${BENCHMARK_VERSION}.tar.gz
+
+for COMPILER in clang gcc; do
+    if [[ "$COMPILER" == "clang" ]]; then
+        export CC=clang-17
+        export CXX=clang++-17
+    elif [[ "$COMPILER" == "gcc" ]]; then
+        export CC=gcc-13
+        export CXX=g++-13
+    else
+        echo "Unknown compiler: $COMPILER"
+        exit 1
+    fi
+
+    cd "$HOME/pkgs"
+    mkdir $COMPILER
+    cd $COMPILER
+
+    INSTALL_DIR=/opt/static-libs/$COMPILER
+
+    tar xf ../libarchive-${LIBARCHIVE_VERSION}.tar.xz
+    cd libarchive-${LIBARCHIVE_VERSION}
+    ./configure --prefix="$INSTALL_DIR" --without-iconv --without-xml2 --without-expat
+    make -j$(nproc)
+    make install
+
+    cd "$HOME/pkgs/$COMPILER"
+    tar xf ../file-${FILE_VERSION}.tar.gz
+    cd file-${FILE_VERSION}
+    ./configure --prefix="$INSTALL_DIR" --enable-static=yes --enable-shared=no
+    make -j$(nproc)
+    make install
+
+    cd "$HOME/pkgs/$COMPILER"
+    tar xf ../flac-${FLAC_VERSION}.tar.xz
+    cd flac-${FLAC_VERSION}
+    ./configure --prefix="$INSTALL_DIR" --enable-static=yes --enable-shared=no --disable-doxygen-docs --disable-ogg --disable-programs --disable-examples
+    make -j$(nproc)
+    make install
+
+    cd "$HOME/pkgs/$COMPILER"
+    tar xf ../libunwind-${LIBUNWIND_VERSION}.tar.gz
+    cd libunwind-${LIBUNWIND_VERSION}
+    ./configure --prefix="$INSTALL_DIR"
+    make -j$(nproc)
+    make install
+
+    # cd "$HOME/pkgs"
+    # tar xf v${BENCHMARK_VERSION}.tar.gz
+    # cd benchmark-${BENCHMARK_VERSION}
+    # mkdir build
+    # cd build
+    # cmake .. -DBENCHMARK_DOWNLOAD_DEPENDENCIES=1 -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR"
+    # make -j$(nproc)
+    # make install
+done
 
 cd "$HOME"
 rm -rf pkgs