services/nomad/build/build-rsyncd,buildbot: improve sync/index of packages

* to prevent re-syncing packages between each package build, use --remove-source-files
* to prevent races with syncing and signing, update the repo as a pre-xfer
  script and collect the log to show in the builder output

Author: classabbyamp

services/nomad/build/build-rsyncd.nomad

@@ -65,8 +65,8 @@ job "build-rsyncd" {
       }
 
       template {
-        data        = file("rsync-post-xfer")
-        destination = "local/rsync-post-xfer"
+        data        = file("rsync-update-repo")
+        destination = "local/rsync-update-repo"
         perms       = "0755"
       }
 
@@ -114,8 +114,13 @@ EOF
 [incoming-${template.value}]
 path = /incoming/${template.value}
 auth users = buildsync:rw
-filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .*
-post-xfer exec = /local/rsync-post-xfer
+filter = + */ + *.${template.value}.xbps - *.sig - *.sig2 - *-repodata* - .* - *.log
+
+[updaterepo-${template.value}]
+path = /incoming/${template.value}
+auth users = buildsync:r
+filter = + update-repo.log - * - .*
+early exec = /local/rsync-update-repo
 EOF
           destination = "local/rsyncd.conf.d/${template.value}.conf.inc"
         }

services/nomad/build/buildbot.cfg

@@ -10,7 +10,6 @@ from pathlib import Path
 from twisted.internet import defer
 from twisted.python import log
 
-from buildbot.process.results import SUCCESS
 from buildbot.plugins import util, secrets, reporters, worker, schedulers
 from buildbot.plugins import steps
 
@@ -222,20 +221,45 @@ def make_rsync_cmd(props):
     return ['bash', '-e', '-c',
             util.Interpolate("""
 echo "=> Syncing packages to the shadow repository..."
-rsync -vurk --delete-after \
+rsync -vurk --delete-after --remove-source-files \
 --filter='+ */' --filter='+ *.%(prop:target)s.xbps' \
 --filter='- .*' --filter='- *' \
 --password-file=/secrets/rsync/password /hostdir/binpkgs/ \
 {{ range nomadService 1 $allocID "build-rsyncd" -}}
 rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s
 {{ end -}}
-case "$?" in
+RSYNC_EXIT="$?"
+echo "=> Cleaning repodatas..."
+find /hostdir/binpkgs -name '%(prop:target)s-repodata' -print -delete
+case "$RSYNC_EXIT" in
     0|23|24) exit 0 ;;
     *) exit 1 ;;
 esac
 """)]
 
 
+@util.renderer
+def make_update_repo_cmd(props):
+    return ['bash', '-e', '-c',
+            util.Interpolate(f"""
+echo "=> Moving packages to the shadow repository..."
+rsync -vurk --delete-after --password-file=/secrets/rsync/password \
+{{ range nomadService 1 $allocID "build-rsyncd" -}}
+rsync://buildsync@{{ .Address }}:{{ .Port }}/updaterepo-%(prop:target)s/update-repo.log ./update-repo.log
+{{ end -}}
+case "$?" in
+    0|23|24)
+        cat ./update-repo.log
+        exit 0
+        ;;
+    *)
+        cat ./update-repo.log
+        exit 1
+        ;;
+esac
+""")]
+
+
 @util.renderer
 def make_clean_cmd(props):
     return [
@@ -259,6 +283,11 @@ def build_packages(props):
             logname=f'sync:{p}',
             haltOnFailure=True,
         ))
+        cmds.append(util.ShellArg(
+            command=make_update_repo_cmd,
+            logname=f'update-repo:{p}',
+            haltOnFailure=True,
+        ))
     if cmds:
         cmds.append(util.ShellArg(
             command=['make', 'clean'],

services/nomad/build/rsync-post-xfer

@@ -0,0 +1,58 @@
+#!/bin/sh
+
+msg() {
+    printf '\033[32m=> %s\033[0m\n' "$*"
+}
+
+process_repo() {
+    repo="$1"
+    msg "Processing repository: $repo"
+    if [ ! -d "$RSYNC_MODULE_PATH/$repo" ] || [ ! -d "$TGT_BASE/$TGT/$repo" ]; then
+        msg "Nothing to do"
+        return
+    fi
+    # add to repodata
+    find "$RSYNC_MODULE_PATH"/"$repo" -mindepth 1 -maxdepth 1 -name '*.xbps' -printf "$TGT_BASE/$TGT/$repo/%f\0" | \
+        xargs -r0 env XBPS_TARGET_ARCH="$ARCH" xbps-rindex -a
+    # clean old packages from repodata
+    XBPS_TARGET_ARCH="$ARCH" xbps-rindex -c "$TGT_BASE/$TGT/$repo"
+    # remove old packages
+    XBPS_TARGET_ARCH="$ARCH" xbps-rindex -r "$TGT_BASE/$TGT/$repo"
+}
+
+run() {
+    set -e
+    TGT_BASE="/mirror/current"
+    ARCH="${RSYNC_MODULE_NAME#*-}"
+
+    case "$ARCH" in
+        aarch64*) TGT="aarch64" ;;
+        *-musl) TGT="musl" ;;
+        *) TGT="" ;;
+    esac
+
+    # copy files to repo
+    msg "Copying new packages to the shadow mirror"
+    rsync -va "${RSYNC_MODULE_PATH:?}"/ "$TGT_BASE/$TGT/"
+
+    for repo in / /bootstrap /debug /nonfree; do
+        process_repo "$repo"
+    done
+
+    case "$ARCH" in
+        x86_64|i686)
+            ARCH=x86_64
+            for repo in /multilib /multilib/bootstrap /multilib/nonfree; do
+                process_repo "$repo"
+            done
+            ;;
+    esac
+
+    # clean up incoming
+    msg "Cleaning up"
+    rm -rvf "${RSYNC_MODULE_PATH:?}"/*.xbps
+    set +e
+}
+
+# if this exits non-zero, rsync will fail and the log will be lost
+run >"$RSYNC_MODULE_PATH"/update-repo.log 2>&1 || true