diff --git a/README.md b/README.md index d2fac18..f513be8 100644 --- a/README.md +++ b/README.md @@ -36,3 +36,4 @@ The script can be configured using environment variables: - `PARALLEL`: Toggle parallel downloads. (Currently only applicable for rsync.) Default: `0` (off) due to the jankiness of this code path. This might be useful to enable for an initial sync, disabling it after. Incremental syncs using this method may be slower than normal. Again, please don't use it outside of initally setting up your mirror. - `MAX_THREADS`: The maximum number of threads to use. Default: output of `nproc` (`$(nproc)`). - `USE_RCLONE`: Use Rclone instead of rsync, mirroring from the HTTP server. Default: `0` (off). The script mirrors from the Fyra Labs rsync server by default. +- `SYNC_SOURCES`: Whether to sync source repositories. Default: `1` (on). diff --git a/mirror.sh b/mirror.sh index 0a6cf5d..f721dc9 100755 --- a/mirror.sh +++ b/mirror.sh @@ -21,10 +21,16 @@ fi : ${MAX_THREADS:=$(nproc)} +: ${SYNC_SOURCES:=1} +if [ "$SYNC_SOURCES" -eq 0 ]; then + RSYNC_OPTS+=("--exclude=*-source/") + RCLONE_OPTS+=("--exclude=*-source/") +fi + list_all_files() { # echo "Fetching file list" # clean up all the folders too, I guess; ^d excludes directories - $RSYNC -rt $MIRROR_URL | grep -v '^d' | awk '{print $5}' | grep -v '/$' | sort | uniq + $RSYNC "${RSYNC_OPTS[@]}" -rt $MIRROR_URL | grep -v '^d' | awk '{print $5}' | grep -v '/$' | sort | uniq } # now, parallelize the rsync using GNU parallel @@ -35,13 +41,13 @@ parallel_rsync() { list_all_files | xargs -P $MAX_THREADS -I '{}' \ $RSYNC \ -avPz --mkpath \ - $RSYNC_OPTS \ + "${RSYNC_OPTS[@]}" \ --delete \ $MIRROR_URL{} \ $MIRROR_DIR/{} # We run rsync again just to get rid of any files that are deleted - $RSYNC -avPzr $RSYNC_OPTS --delete $MIRROR_URL $MIRROR_DIR + $RSYNC -avPzr "${RSYNC_OPTS[@]}" --delete $MIRROR_URL $MIRROR_DIR } echo "Mirroring $MIRROR_URL to $MIRROR_DIR" @@ -53,7 +59,7 @@ if [ "$USE_RCLONE" -eq 0 ]; then if [ $PARALLEL -eq 1 ]; then parallel_rsync else - $RSYNC -avPzr $RSYNC_OPTS --delete $MIRROR_URL $MIRROR_DIR + $RSYNC -avPzr "${RSYNC_OPTS[@]}" --delete $MIRROR_URL $MIRROR_DIR fi else @@ -67,7 +73,7 @@ else --transfers $MAX_THREADS \ --stats 5s \ --stats-one-line \ - -v $RCLONE_OPTS \ + -v "${RCLONE_OPTS[@]}" \ --http-url $DEFAULT_HTTP_MIRROR \ :http:/ $MIRROR_DIR fi