Merge pull request #1 from terrapkg/exclude-sources

Add option to exclude syncing source repos

README.md

@@ -36,3 +36,4 @@ The script can be configured using environment variables:
 - `PARALLEL`: Toggle parallel downloads. (Currently only applicable for rsync.) Default: `0` (off) due to the jankiness of this code path. This might be useful to enable for an initial sync, disabling it after. Incremental syncs using this method may be slower than normal. Again, please don't use it outside of initally setting up your mirror.
 - `MAX_THREADS`: The maximum number of threads to use. Default: output of `nproc` (`$(nproc)`).
 - `USE_RCLONE`: Use Rclone instead of rsync, mirroring from the HTTP server. Default: `0` (off). The script mirrors from the Fyra Labs rsync server by default.
+- `SYNC_SOURCES`: Whether to sync source repositories. Default: `1` (on).

mirror.sh

@@ -21,10 +21,16 @@ fi
 
 : ${MAX_THREADS:=$(nproc)}
 
+: ${SYNC_SOURCES:=1}
+if [ "$SYNC_SOURCES" -eq 0 ]; then
+    RSYNC_OPTS+=("--exclude=*-source/")
+    RCLONE_OPTS+=("--exclude=*-source/")
+fi
+
 list_all_files() {
     # echo "Fetching file list"
     # clean up all the folders too, I guess; ^d excludes directories
-    $RSYNC -rt $MIRROR_URL | grep -v '^d' | awk '{print $5}' | grep -v '/$' | sort | uniq
+    $RSYNC "${RSYNC_OPTS[@]}" -rt $MIRROR_URL | grep -v '^d' | awk '{print $5}' | grep -v '/$' | sort | uniq
 }
 
 # now, parallelize the rsync using GNU parallel
@@ -35,13 +41,13 @@ parallel_rsync() {
     list_all_files | xargs -P $MAX_THREADS -I '{}' \
         $RSYNC \
         -avPz --mkpath \
-        $RSYNC_OPTS \
+        "${RSYNC_OPTS[@]}" \
         --delete \
         $MIRROR_URL{} \
         $MIRROR_DIR/{}
 
     # We run rsync again just to get rid of any files that are deleted
-    $RSYNC -avPzr $RSYNC_OPTS --delete $MIRROR_URL $MIRROR_DIR
+    $RSYNC -avPzr "${RSYNC_OPTS[@]}" --delete $MIRROR_URL $MIRROR_DIR
 }
 
 echo "Mirroring $MIRROR_URL to $MIRROR_DIR"
@@ -53,7 +59,7 @@ if [ "$USE_RCLONE" -eq 0 ]; then
     if [ $PARALLEL -eq 1 ]; then
         parallel_rsync
     else
-        $RSYNC -avPzr $RSYNC_OPTS --delete $MIRROR_URL $MIRROR_DIR
+        $RSYNC -avPzr "${RSYNC_OPTS[@]}" --delete $MIRROR_URL $MIRROR_DIR
     fi
 
 else
@@ -67,7 +73,7 @@ else
         --transfers $MAX_THREADS \
         --stats 5s \
         --stats-one-line \
-        -v $RCLONE_OPTS \
+        -v "${RCLONE_OPTS[@]}" \
         --http-url $DEFAULT_HTTP_MIRROR \
         :http:/ $MIRROR_DIR
 fi