fix: wait for the bootstrap lock without a command timeout; safe unlock

From the local CodeRabbit review:
- A replica waiting for the lock used the default 30 s command timeout and
  failed to boot while another replica seeded a fresh database. Wait with
  no timeout, logging that it is waiting.
- An unlock failure in the finally (e.g. the connection already broken)
  replaced the real bootstrap error and skipped closing the connection.
  Log it instead, and always close.

DorkNet.Server/Startup/DatabaseBootstrap.cs

@@ -15,9 +15,9 @@ namespace DorkNet.Server.Startup;
 ///   <item>SQLite — <c>Database.Migrate()</c> after
 ///   <see cref="BaselineExistingSchemaIfNeeded"/> reconciles any legacy
 ///   migration-history state.</item>
-///   <item>Postgres — <c>EnsureCreated</c> behind a transaction-scoped
+///   <item>Postgres — <c>EnsureCreated</c>, with the whole bootstrap
-///   advisory lock so concurrent replica boots don't race on
+///   (schema and seeds) behind a session advisory lock so concurrent
-///   <c>CREATE TABLE</c>. Connect retries cover the
+///   replica boots take turns instead of racing. Connect retries cover the
 ///   container-orchestrated boot window where Postgres DNS isn't
 ///   resolvable yet.</item>
 /// </list>
@@ -40,21 +40,23 @@ public static class DatabaseBootstrap
             await ConfigureSqliteForLocalConcurrencyAsync(db);
             BaselineExistingSchemaIfNeeded(db);
             db.Database.Migrate();
+            await RunUpgradesAndSeedsAsync(app, scope, db);
+            return;
         }
-        else
+
-        {
         // Postgres path: schema generated from the EF Core model via
-            // EnsureCreated. Wrapped in a transaction-scoped advisory lock
+        // EnsureCreated, then the same upgrades and seeds as SQLite. The
-            // so concurrent replicas booting against the same database
+        // whole bootstrap holds a session-level advisory lock, so replicas
-            // don't race on CREATE TABLE — only the first replica to take
+        // booting against one database take turns: the first creates the
-            // the lock runs EnsureCreated, the rest see "schema already
+        // schema and seeds, the rest find everything in place. Holding it
-            // exists" and proceed.
+        // only around EnsureCreated left the seeders racing (duplicate keys
+        // on a fresh database).
         //
-            // Lock key 0x444F524B (ascii "DORK") is hard-coded; it just
+        // Lock key 0x444F524B (ascii "DORK") is hard-coded; it just needs
-            // needs to be the same int64 across replicas and not collide
+        // to be the same int64 across replicas and not collide with
-            // with anything else in the database. pg_advisory_xact_lock
+        // anything else in the database. It is released in the finally
-            // releases automatically on COMMIT, so a crashed replica can't
+        // below, and Postgres drops it if the replica's connection dies, so
-            // wedge the next boot.
+        // a crashed replica can't wedge the next boot.
         var conn = db.Database.GetDbConnection();
 
         // Retry initial Postgres connect — on container-orchestrated boots
@@ -82,19 +84,66 @@ public static class DatabaseBootstrap
                 await Task.Delay(delay);
             }
         }
-            await using var tx = await conn.BeginTransactionAsync();
+
-            await using (var lockCmd = conn.CreateCommand())
+        // The connection stays open until the finally, so every step below
+        // (they all share this scope's DbContext) runs on the session that
+        // holds the lock. Waiting for it has no command timeout: another
+        // replica may be seeding a fresh database (thumbnail downloads
+        // included) for well over the default 30 s.
+        connectLogger.LogInformation("[bootstrap] waiting for the database bootstrap lock");
+        var previousTimeout = db.Database.GetCommandTimeout();
+        db.Database.SetCommandTimeout(0);
+        try
         {
-                lockCmd.Transaction = tx;
+            await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_lock(1146246987);");
-                lockCmd.CommandText = "SELECT pg_advisory_xact_lock(1146246987);";
-                await lockCmd.ExecuteNonQueryAsync();
         }
+        finally
+        {
+            db.Database.SetCommandTimeout(previousTimeout);
+        }
+        connectLogger.LogInformation("[bootstrap] holding the database bootstrap lock");
+        try
+        {
+            // Begin the transaction through EF, not on the raw connection:
+            // EnsureCreated runs its commands through the migration executor,
+            // which starts its own transaction unless EF already has one, and
+            // fails on a fresh database with "A transaction is already in
+            // progress" when the open one is invisible to it.
+            await using (var tx = await db.Database.BeginTransactionAsync())
+            {
                 db.Database.EnsureCreated();
                 await tx.CommitAsync();
-
+            }
+            await RunUpgradesAndSeedsAsync(app, scope, db);
+        }
+        finally
+        {
+            // Explicit: Npgsql pools connections, so closing alone would not
+            // end the session that holds the lock. If the connection is
+            // already broken the unlock fails too; that must not replace the
+            // real bootstrap error, and the connection still gets closed
+            // (Postgres then drops the lock with the session).
+            try
+            {
+                await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_unlock(1146246987);");
+            }
+            catch (Exception ex)
+            {
+                connectLogger.LogWarning(ex, "[bootstrap] releasing the database bootstrap lock failed");
+            }
+            finally
+            {
                 await conn.CloseAsync();
             }
+        }
+    }
 
+    /// <summary>The idempotent post-schema sequence both providers run:
+    /// column/table upgrades, system account and backfills, legacy data
+    /// transforms, canonical rooms and the per-feature seeders. Flips the
+    /// <c>/healthz</c> readiness flag at the end.</summary>
+    private static async Task RunUpgradesAndSeedsAsync(WebApplication app, IServiceScope scope, DorkNetDbContext db)
+    {
         // Signup-code + pending-device tables. These post-date the
         // consolidated Initial migration, and the Postgres path is
         // EnsureCreated-only (no migration replay), so neither provider

Jenkinsfile

@@ -1,17 +1,20 @@
 // DorkNet client builds — PC (patcher zip + full client) and Quest (.xapk).
 //
-// Runs on the Jenkins image from ci/jenkins (the stock image plus Python with
+// Runs on the "dorknet-builder" agent from ci/jenkins (an inbound agent with
-// LIEF, .NET, the Android NDK/build-tools, apktool, restic and rclone).
+// Python + LIEF, .NET, the Android NDK/build-tools, apktool, restic and rclone).
 //
 // Stock builds come out of the restic build archive on the FTP. Where it lives
 // is server configuration, not part of the repo: the FTP host, login and the
 // restic password are Jenkins credentials, and the repository folders come
-// from the Jenkins container's environment (ARCHIVE_PC_REPO /
+// from the agent's environment (ARCHIVE_PC_REPO / ARCHIVE_QUEST_REPO), each
-// ARCHIVE_QUEST_REPO), each overridable per run. A build is picked by
+// overridable per run. A build is picked by snapshot tags.
-// snapshot tags.
 //
 // On a push, only the platforms whose code changed are rebuilt. A manual run
 // picks the platforms with PLATFORMS. See ci/jenkins/README.md.
+//
+// Finished builds are renamed for players and published to the downloads bucket
+// (ci/downloads) unless PUBLISH_DOWNLOADS is off. The build scripts keep their
+// own file names, which the launcher depends on.
 
 // The builds this branch targets. Any other archived build can be picked per
 // run with the *_SNAPSHOT_TAGS parameters.
@@ -19,11 +22,16 @@ def PC_VERSION_KEY = 'march_2023_03_21'
 def PC_DEFAULT_TAGS = 'manifest=7490748483298966814'   // Steam 2023-03-21 (appVersion 20230317)
 def QUEST_DEFAULT_TAGS = 'version-code=2128210'        // Quest 20230317
 
+// How this branch's builds are listed on the site's Downloads page.
+def RELEASE_ID = '2023.03'
+def RELEASE_TITLE = 'March 2023'
+def RELEASE_GAME_VERSION = '2023-03-21'
+
 def buildPc = false
 def buildQuest = false
 
 pipeline {
-  agent any
+  agent { label 'dorknet-builder' }
 
   options {
     timestamps()
@@ -45,11 +53,13 @@ pipeline {
     string(name: 'QUEST_SNAPSHOT_TAGS', defaultValue: '',
       description: "restic tags picking the stock Quest build, comma = AND. Empty: ${QUEST_DEFAULT_TAGS}")
     string(name: 'PC_REPO', defaultValue: '',
-      description: 'PC restic repository folder on the FTP. Empty: ARCHIVE_PC_REPO from the Jenkins environment.')
+      description: 'PC restic repository folder on the FTP. Empty: ARCHIVE_PC_REPO set on the agent.')
     string(name: 'QUEST_REPO', defaultValue: '',
-      description: 'Quest restic repository folder on the FTP. Empty: ARCHIVE_QUEST_REPO from the Jenkins environment.')
+      description: 'Quest restic repository folder on the FTP. Empty: ARCHIVE_QUEST_REPO set on the agent.')
     booleanParam(name: 'LIST_SNAPSHOTS', defaultValue: false,
       description: 'Only list the snapshots matching the tags in both repositories (to find a build), then stop.')
+    booleanParam(name: 'PUBLISH_DOWNLOADS', defaultValue: true,
+      description: 'Publish what this run built to the downloads bucket (listed on the site Downloads page). Push builds publish too.')
   }
 
   stages {
@@ -88,8 +98,8 @@ pipeline {
                    "PC_TAGS=${params.PC_SNAPSHOT_TAGS}", "QUEST_TAGS=${params.QUEST_SNAPSHOT_TAGS}"]) {
             sh '''
               export RESTIC_CACHE_DIR="$DORKNET_CACHE/restic"
-              pc="${PC_REPO_OVERRIDE:-${ARCHIVE_PC_REPO:?set ARCHIVE_PC_REPO on Jenkins or PC_REPO on the run}}"
+              pc="${PC_REPO_OVERRIDE:-${ARCHIVE_PC_REPO:?set ARCHIVE_PC_REPO on the agent or PC_REPO on the run}}"
-              quest="${QUEST_REPO_OVERRIDE:-${ARCHIVE_QUEST_REPO:?set ARCHIVE_QUEST_REPO on Jenkins or QUEST_REPO on the run}}"
+              quest="${QUEST_REPO_OVERRIDE:-${ARCHIVE_QUEST_REPO:?set ARCHIVE_QUEST_REPO on the agent or QUEST_REPO on the run}}"
               echo "== PC: $pc";       archive-restore list "$pc" "$PC_TAGS"
               echo "== Quest: $quest"; archive-restore list "$quest" "$QUEST_TAGS"
             '''
@@ -113,7 +123,7 @@ pipeline {
             withEnv(["REPO_OVERRIDE=${params.PC_REPO}", "TAGS=${params.PC_SNAPSHOT_TAGS ?: PC_DEFAULT_TAGS}"]) {
               sh(returnStdout: true, script: '''
                 export RESTIC_CACHE_DIR="$DORKNET_CACHE/restic"
-                repo="${REPO_OVERRIDE:-${ARCHIVE_PC_REPO:?set ARCHIVE_PC_REPO on Jenkins or PC_REPO on the run}}"
+                repo="${REPO_OVERRIDE:-${ARCHIVE_PC_REPO:?set ARCHIVE_PC_REPO on the agent or PC_REPO on the run}}"
                 archive-restore restore "$repo" "$TAGS" "$DORKNET_CACHE/pc" --exclude /.DepotDownloader
               ''').trim().readLines().last()
             }
@@ -141,7 +151,7 @@ pipeline {
             withEnv(["REPO_OVERRIDE=${params.QUEST_REPO}", "TAGS=${params.QUEST_SNAPSHOT_TAGS ?: QUEST_DEFAULT_TAGS}"]) {
               sh(returnStdout: true, script: '''
                 export RESTIC_CACHE_DIR="$DORKNET_CACHE/restic"
-                repo="${REPO_OVERRIDE:-${ARCHIVE_QUEST_REPO:?set ARCHIVE_QUEST_REPO on Jenkins or QUEST_REPO on the run}}"
+                repo="${REPO_OVERRIDE:-${ARCHIVE_QUEST_REPO:?set ARCHIVE_QUEST_REPO on the agent or QUEST_REPO on the run}}"
                 archive-restore restore "$repo" "$TAGS" "$DORKNET_CACHE/quest"
               ''').trim().readLines().last()
             }
@@ -179,15 +189,39 @@ pipeline {
         }
       }
     }
+
+    stage('Downloads') {
+      when { expression { buildPc || buildQuest } }
+      steps {
+        sh """
+          python3 ci/downloads/publish.py stage --out out --release ${RELEASE_ID} \\
+            --title '${RELEASE_TITLE}' --game-version ${RELEASE_GAME_VERSION} --version-key ${PC_VERSION_KEY}
+        """
+        script {
+          if (params.PUBLISH_DOWNLOADS) {
+            // The bucket's S3 endpoint and name come from the agent's
+            // environment (DOWNLOADS_S3_ENDPOINT / DOWNLOADS_BUCKET).
+            withCredentials([usernamePassword(credentialsId: 'downloads-s3',
+                usernameVariable: 'DOWNLOADS_S3_KEY_ID', passwordVariable: 'DOWNLOADS_S3_SECRET')]) {
+              sh 'python3 ci/downloads/publish.py upload --staged out/downloads'
+            }
+          } else {
+            echo 'PUBLISH_DOWNLOADS is off: the files are only archived.'
+          }
+        }
+      }
+    }
   }
 
   post {
     success {
-      archiveArtifacts artifacts: 'out/**', allowEmptyArchive: true, fingerprint: true
+      // The renamed, player-facing files (and release.json): the same set the
+      // downloads page offers.
+      archiveArtifacts artifacts: 'out/downloads/**', allowEmptyArchive: true, fingerprint: true
     }
     cleanup {
       // Runs after archiving. Keeps the restic cache and restored stock
-      // builds ($DORKNET_CACHE in jenkins_home, outside the workspace); drops
+      // builds ($DORKNET_CACHE on the agent, outside the workspace); drops
       // the workspace's build output and scratch.
       cleanWs(deleteDirs: true, notFailBuild: true)
     }

ci/downloads/publish.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python3
+"""Publish DorkNet client builds to the downloads bucket.
+
+  stage    the build output in out/ -> out/downloads/<release>/ under the
+           names players see, plus release.json describing those files
+  upload   that folder -> the downloads bucket, then merge release.json into
+           the bucket's manifest.json
+
+The DorkNet server reads the bucket and offers the files on its site (the
+Downloads page, and Latest build on the Quest page); see docs/deploy.md.
+
+The build scripts keep their own file names on purpose: the launcher fetches
+dorknet-clientpatch-<VersionKey>.zip from GitHub Releases by that exact name.
+Only the published copies are renamed:
+
+    dorknet-client-<key>.zip              DorkNet-<release>-PC-Full.zip
+    dorknet-clientpatch-<key>.zip         DorkNet-<release>-PC-Patch.zip
+    DorkNet-Quest-<ver>/DorkNet-<ver>.xapk  DorkNet-<release>-Quest.xapk
+    DorkNet-Quest-<ver>/install.*, README DorkNet-<release>-Quest-Installer.zip
+
+The server reads the bucket over plain HTTP on the internal network (e.g.
+Garage's web endpoint); nothing needs to be public. upload talks to its S3
+API with rclone; the endpoint, bucket and key come from the environment:
+
+    DOWNLOADS_S3_ENDPOINT   S3 API URL, e.g. http://garage-<uuid>:3900
+    DOWNLOADS_BUCKET        bucket name, e.g. dorknet-downloads
+    DOWNLOADS_S3_REGION     optional, default garage
+    DOWNLOADS_S3_KEY_ID / DOWNLOADS_S3_SECRET
+
+    python3 ci/downloads/publish.py stage --out out --release 2023.03 \\
+        --title "March 2023" --game-version 2023-03-21 --version-key march_2023_03_21
+    python3 ci/downloads/publish.py upload --staged out/downloads
+"""
+from __future__ import annotations
+
+import argparse
+import datetime
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+from typing import NoReturn
+
+NO_CACHE = "Cache-Control: no-cache"
+
+INSTALLER_NOTE = """Unzip these files into the same folder as {xapk}, then
+run install.cmd (Windows) or ./install.sh (macOS / Linux).
+
+"""
+
+
+def fail(msg: str) -> NoReturn:
+    print(f"error: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def step(msg: str) -> None:
+    print(f"==> {msg}", flush=True)
+
+
+def human(n: int) -> str:
+    return f"{n / 2**20:,.1f} MB" if n >= 2**20 else f"{max(1, round(n / 2**10))} KB"
+
+
+def sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        while chunk := f.read(1 << 20):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def link_or_copy(src: Path, dst: Path) -> None:
+    """Hard-link where possible: the full client is several GB."""
+    dst.unlink(missing_ok=True)
+    try:
+        os.link(src, dst)
+    except OSError:
+        shutil.copy2(src, dst)
+
+
+def game_exe(full_zip: Path) -> str | None:
+    """The game's exe: the one with a matching <name>_Data folder at the root."""
+    with zipfile.ZipFile(full_zip) as z:
+        names = z.namelist()
+    tops = {n.split("/", 1)[0] for n in names}
+    exes = sorted(t for t in tops if t.lower().endswith(".exe") and f"{t[:-4]}_Data" in tops)
+    return exes[0] if exes else None
+
+
+# ---- stage ----------------------------------------------------------------
+
+def cmd_stage(a: argparse.Namespace) -> None:
+    out = Path(a.out)
+    dest = Path(a.dest) if a.dest else out / "downloads"
+    rel_dir = dest / a.release
+    if rel_dir.exists():
+        shutil.rmtree(rel_dir)
+    rel_dir.mkdir(parents=True)
+    prefix = f"DorkNet-{a.release}"
+
+    staged: list[tuple[str, str, Path]] = []  # (id, platform, staged file)
+    extra: dict[str, str] = {}
+
+    full = out / "pc" / f"dorknet-client-{a.version_key}.zip"
+    patch = out / "pc" / f"dorknet-clientpatch-{a.version_key}.zip"
+    if full.is_file():
+        dst = rel_dir / f"{prefix}-PC-Full.zip"
+        link_or_copy(full, dst)
+        staged.append(("pc-full", "pc", dst))
+        exe = game_exe(full)
+        if exe:
+            extra["pc_exe"] = exe
+    if patch.is_file():
+        dst = rel_dir / f"{prefix}-PC-Patch.zip"
+        link_or_copy(patch, dst)
+        staged.append(("pc-patch", "pc", dst))
+
+    packages = sorted((out / "quest").glob("DorkNet-Quest-*/"))
+    if len(packages) > 1:
+        fail(f"more than one Quest package folder in {out / 'quest'}: {', '.join(p.name for p in packages)}")
+    if packages:
+        pkg = packages[0]
+        xapks = sorted(pkg.glob("*.xapk"))
+        if len(xapks) != 1:
+            fail(f"expected one .xapk in {pkg}, found {len(xapks)}")
+        xapk = rel_dir / f"{prefix}-Quest.xapk"
+        link_or_copy(xapks[0], xapk)
+        staged.append(("quest", "quest", xapk))
+
+        installer = rel_dir / f"{prefix}-Quest-Installer.zip"
+        with zipfile.ZipFile(installer, "w", zipfile.ZIP_DEFLATED) as z:
+            for name in ("install.cmd", "install.ps1", "install.sh"):
+                src = pkg / name
+                if not src.is_file():
+                    fail(f"{src} missing")
+                info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
+                info.external_attr = (0o755 if name.endswith(".sh") else 0o644) << 16
+                info.compress_type = zipfile.ZIP_DEFLATED
+                z.writestr(info, src.read_bytes())
+            readme = pkg / "README.txt"
+            text = INSTALLER_NOTE.format(xapk=xapk.name) + (readme.read_text(encoding="utf-8") if readme.is_file() else "")
+            info = zipfile.ZipInfo("README.txt", date_time=(1980, 1, 1, 0, 0, 0))
+            info.compress_type = zipfile.ZIP_DEFLATED
+            z.writestr(info, text)
+        staged.append(("quest-installer", "quest", installer))
+
+    if not staged:
+        fail(f"nothing to publish in {out}")
+
+    built_at = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()
+    files = []
+    for fid, platform, path in staged:
+        step(f"{path.name}: hashing")
+        files.append({
+            "id": fid,
+            "platform": platform,
+            "name": path.name,
+            "path": f"{a.release}/{path.name}",
+            "size": path.stat().st_size,
+            "sha256": sha256(path),
+            "build": a.build,
+            "commit": a.commit,
+            "built_at": built_at,
+        })
+    release = {"id": a.release, "title": a.title, "game_version": a.game_version, **extra, "files": files}
+    (dest / "release.json").write_text(json.dumps(release, indent=2) + "\n", encoding="utf-8")
+    print()
+    for f in files:
+        print(f"  {f['name']}  {human(f['size'])}  {f['sha256'][:16]}…")
+    print(f"Staged {len(files)} file(s) in {rel_dir}")
+
+
+# ---- upload ---------------------------------------------------------------
+
+def rclone_env() -> dict[str, str]:
+    env = dict(os.environ)
+    for k in ("DOWNLOADS_S3_ENDPOINT", "DOWNLOADS_BUCKET", "DOWNLOADS_S3_KEY_ID", "DOWNLOADS_S3_SECRET"):
+        if not env.get(k):
+            fail(f"{k} is not set")
+    env.update({
+        "RCLONE_CONFIG_DL_TYPE": "s3",
+        "RCLONE_CONFIG_DL_PROVIDER": "Other",
+        "RCLONE_CONFIG_DL_ENDPOINT": env["DOWNLOADS_S3_ENDPOINT"],
+        "RCLONE_CONFIG_DL_REGION": env.get("DOWNLOADS_S3_REGION") or "garage",
+        "RCLONE_CONFIG_DL_ACCESS_KEY_ID": env["DOWNLOADS_S3_KEY_ID"],
+        "RCLONE_CONFIG_DL_SECRET_ACCESS_KEY": env["DOWNLOADS_S3_SECRET"],
+        "RCLONE_CONFIG_DL_FORCE_PATH_STYLE": "true",
+        "RCLONE_CONFIG_DL_NO_CHECK_BUCKET": "true",
+    })
+    return env
+
+
+def rclone(env: dict[str, str], *args: str, stdin: str | None = None, check: bool = True) -> subprocess.CompletedProcess:
+    r = subprocess.run(["rclone", *args], env=env, input=stdin, capture_output=True, text=True)
+    if check and r.returncode != 0:
+        fail(f"rclone {args[0]} failed ({r.returncode}): {r.stderr.strip()}")
+    return r
+
+
+def merge(manifest: dict, release: dict) -> dict:
+    """Replace this release's files that were just built; keep the others
+    (a push build may rebuild only one platform)."""
+    releases = [r for r in manifest.get("releases", []) if r.get("id") != release["id"]]
+    old = next((r for r in manifest.get("releases", []) if r.get("id") == release["id"]), {})
+    new_ids = {f["id"] for f in release["files"]}
+    kept = [f for f in old.get("files", []) if f.get("id") not in new_ids]
+    merged = {**old, **release, "files": kept + release["files"]}
+    if "pc_exe" not in release and "pc_exe" in old:
+        merged["pc_exe"] = old["pc_exe"]
+    order = ["pc-full", "pc-patch", "quest", "quest-installer"]
+    merged["files"].sort(key=lambda f: order.index(f["id"]) if f["id"] in order else len(order))
+    releases.append(merged)
+    releases.sort(key=lambda r: r["id"], reverse=True)
+    out = {**manifest, "releases": releases,
+           "updated": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()}
+    return out
+
+
+def cmd_upload(a: argparse.Namespace) -> None:
+    staged = Path(a.staged)
+    release = json.loads((staged / "release.json").read_text(encoding="utf-8"))
+    env = rclone_env()
+    bucket = f"dl:{env['DOWNLOADS_BUCKET']}"
+    rel = release["id"]
+
+    # 1. the files, before the manifest that points at them
+    for f in release["files"]:
+        step(f"uploading {f['name']} ({human(f['size'])})")
+        rclone(env, "copyto", str(staged / f["path"]), f"{bucket}/{f['path']}",
+               "--header-upload", NO_CACHE, "--s3-upload-concurrency", "8", "--s3-chunk-size", "64M")
+
+    # 2. merge into the live manifest
+    r = rclone(env, "cat", f"{bucket}/manifest.json", check=False)
+    manifest = json.loads(r.stdout) if r.returncode == 0 and r.stdout.strip() else {}
+    manifest = merge(manifest, release)
+
+    # 3. the manifest, last: it is what makes the new files visible
+    step("uploading manifest.json")
+    rclone(env, "rcat", f"{bucket}/manifest.json", "--header-upload", NO_CACHE,
+           "--header-upload", "Content-Type: application/json", stdin=json.dumps(manifest, indent=2) + "\n")
+
+    # 4. drop files of this release the manifest no longer lists (renames)
+    listed = {f["path"] for r in manifest["releases"] if r["id"] == rel for f in r["files"]}
+    r = rclone(env, "lsf", f"{bucket}/{rel}/", "--files-only", check=False)
+    for name in r.stdout.split():
+        if f"{rel}/{name}" not in listed:
+            step(f"removing stale {rel}/{name}")
+            rclone(env, "deletefile", f"{bucket}/{rel}/{name}")
+    print(f"Published {len(release['files'])} file(s) for {release['title']}.")
+
+
+# ---- cli ------------------------------------------------------------------
+
+def main() -> None:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("stage", help="out/ -> out/downloads/<release>/ + release.json")
+    s.add_argument("--out", default="out", help="the build output folder (default: out)")
+    s.add_argument("--dest", help="where to stage (default: <out>/downloads)")
+    s.add_argument("--release", required=True, help="release id used in file names and paths, e.g. 2023.03")
+    s.add_argument("--title", required=True, help='shown on the Downloads page, e.g. "March 2023"')
+    s.add_argument("--game-version", required=True, help="the stock build's date, e.g. 2023-03-21")
+    s.add_argument("--version-key", required=True, help="the PC build's version key, e.g. march_2023_03_21")
+    s.add_argument("--build", default=os.environ.get("BUILD_NUMBER", ""), help="CI build number (default: $BUILD_NUMBER)")
+    s.add_argument("--commit", default=os.environ.get("GIT_COMMIT", ""), help="source commit (default: $GIT_COMMIT)")
+    s.set_defaults(fn=cmd_stage)
+
+    u = sub.add_parser("upload", help="staged folder -> downloads bucket + manifest.json")
+    u.add_argument("--staged", default="out/downloads", help="the folder `stage` wrote (default: out/downloads)")
+    u.set_defaults(fn=cmd_upload)
+
+    a = p.parse_args()
+    a.fn(a)
+
+
+if __name__ == "__main__":
+    main()

ci/jenkins/Dockerfile

@@ -1,7 +1,7 @@
-# DorkNet Jenkins: the stock Jenkins image plus everything the PC and Quest
+# DorkNet build agent: a Jenkins inbound agent with everything the PC and
-# client builds need, so the job (the Jenkinsfile at the repo root) runs right
+# Quest client builds need. It runs next to Jenkins on Coolify, connects to
-# on the controller. Jobs, plugins and settings stay in the jenkins_home
+# the controller over WebSocket, and runs the jobs labelled "dorknet-builder"
-# volume; only the image changes.
+# (the Jenkinsfile at the repo root). Jenkins itself stays on the stock image.
 #
 #   PC      Python + .NET SDK          tools/build_pc.py
 #   Quest   Python/LIEF + JDK + Android NDK, build-tools, apktool
@@ -9,13 +9,13 @@
 #   Builds  restic + rclone            stock builds restored from the restic
 #                                      archive on the FTP (archive-restore.sh)
 #
+# Java 25: Jenkins supports it until Sep 2029 (Java 21 only until Sep 2027).
+# Keep the agent's Java the same major version as the controller's.
+#
 # See ci/jenkins/README.md.
 
-# Match this to the Jenkins image your Dokploy service runs today, so the
+ARG AGENT_IMAGE=jenkins/inbound-agent:latest-jdk25
-# upgrade keeps the same Jenkins version and Java: e.g.
+FROM ${AGENT_IMAGE}
-#   --build-arg JENKINS_IMAGE=jenkins/jenkins:2.492.3-lts-jdk17
-ARG JENKINS_IMAGE=jenkins/jenkins:lts-jdk17
-FROM ${JENKINS_IMAGE}
 
 USER root
 
@@ -73,9 +73,10 @@ RUN wget -q https://github.com/restic/restic/releases/download/v${RESTIC}/restic
 COPY archive-restore.sh /usr/local/bin/archive-restore
 RUN chmod +x /usr/local/bin/archive-restore
 
-# Restored stock builds and the restic cache live in the persistent
+# Restored stock builds and the restic cache. Mount a volume here so each
-# jenkins_home volume, so each multi-GB build is downloaded once.
+# multi-GB stock build is downloaded once, not after every restart.
-ENV DORKNET_CACHE=/var/jenkins_home/dorknet-cache
+ENV DORKNET_CACHE=/home/jenkins/dorknet-cache
+RUN mkdir -p ${DORKNET_CACHE} && chown jenkins:jenkins ${DORKNET_CACHE}
 
 ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 \
     DOTNET_NOLOGO=1

ci/jenkins/README.md

@@ -3,21 +3,34 @@
 The `Jenkinsfile` at the repo root builds the DorkNet clients on your existing
 Jenkins.
 
-It runs right on the Jenkins container, so that container's image needs the
+The builds run on a **build agent**, not on Jenkins itself:
-build toolchain: Python with LIEF, the .NET SDK, the Android NDK and build-tools,
+- **The agent:** a container built from `ci/jenkins/Dockerfile`, which is the
-apktool, restic and rclone. `ci/jenkins/Dockerfile` is the stock
+  Jenkins inbound agent plus Python with LIEF, the .NET SDK, the Android NDK
-Jenkins image with that toolchain added. Your jobs, plugins and settings live
+  and build-tools, apktool, restic and rclone. It runs on Coolify next to
-in the `jenkins_home` volume, not in the image, so switching images leaves
+  Jenkins and connects to it over WebSocket on Docker's internal network. The
-them alone.
+  job runs on it through the `dorknet-builder` label.
+- **Jenkins itself:** stays on the stock image and runs no builds.
 
-| Platform | Output (Jenkins artifacts) | How |
+Each build is published to the **downloads bucket** under a name players can
+read, and the same files are kept as Jenkins artifacts
+(`out/downloads/2023.03/`). The DorkNet server offers them on its site: the
+**Downloads** page, and **Latest build** on the **Quest** page
+(`docs/deploy.md` → Client downloads).
+
+| Download | What it is | Built from |
 |---|---|---|
-| PC | `out/pc/dorknet-clientpatch-march_2023_03_21.zip`: MelonLoader, mod, config template, Steam stub, `manifest.json`; no game files | `tools/build_pc.py patcher` |
+| `DorkNet-2023.03-PC-Full.zip` | the stock Steam build with the patcher applied and your config; ready to run | `tools/build_pc.py client` → `dorknet-client-march_2023_03_21.zip` |
-| PC | `out/pc/dorknet-client-march_2023_03_21.zip`: the stock Steam build with the patcher applied and your config; ready to run | `tools/build_pc.py client` |
+| `DorkNet-2023.03-PC-Patch.zip` | MelonLoader, mod, config template, Steam stub, `manifest.json`; no game files | `tools/build_pc.py patcher` → `dorknet-clientpatch-march_2023_03_21.zip` |
-| Quest | `out/quest/dorknet-recroom-<code>.apk` and `out/quest/DorkNet-Quest-<ver>/` (`.xapk` + installers) | `DorkNet.Quest/scripts/build.py build` |
+| `DorkNet-2023.03-Quest.xapk` | patched APK + OBB, for the browser installer or SideQuest | `DorkNet.Quest/scripts/build.py build` → `DorkNet-Quest-<ver>/DorkNet-<ver>.xapk` |
+| `DorkNet-2023.03-Quest-Installer.zip` | `install.cmd` / `install.ps1` / `install.sh` + README, unzipped next to the `.xapk` | the same package folder |
 
-The full PC client and the Quest packages contain game files, so keep them on
+The build scripts keep their own names: the launcher downloads
-Jenkins. Don't publish them anywhere public.
+`dorknet-clientpatch-<VersionKey>.zip` from GitHub Releases by that exact
+name. `ci/downloads/publish.py` renames only the published copies.
+
+The full PC client and the Quest packages contain game files. Keep the bucket
+internal and the site behind an access proxy; don't publish them anywhere
+public.
 
 ## Where the stock builds come from
 
@@ -37,8 +50,8 @@ The job defaults to the builds this branch targets: Steam manifest
 build can be chosen per run. Tick **LIST_SNAPSHOTS** to print the snapshots
 matching a filter without building anything.
 
-A restored snapshot is cached in `jenkins_home/dorknet-cache`, so each stock
+A restored snapshot is cached on the agent's `dorknet_cache` volume, so each
-build is downloaded once.
+stock build is downloaded once.
 
 A Quest build needs a resolved hook table for its versionCode, committed under
 `DorkNet.Quest/targets/`. Only `20230317-2128210.json` exists today. The build
@@ -53,36 +66,91 @@ Resolve a table for another build locally with `build.py build --dump-cs`.
   - `Jenkinsfile` and `ci/jenkins/` rebuild both.
 - **Build with Parameters** runs a manual build:
   - `PLATFORMS` (both / pc / quest), and `PC_FULL_CLIENT` for the full PC client;
-  - snapshot tags, and per-run repository overrides.
+  - snapshot tags, and per-run repository overrides;
+  - `PUBLISH_DOWNLOADS` (on by default) to publish to the downloads bucket.
+
+Push builds publish too, but only the platform they rebuilt: the manifest
+keeps the other platform's last files.
 
 ## Setup
 
-### 1. Switch your Jenkins to this image
+### 1. Jenkins itself
 
-On your Jenkins service in Dokploy:
+The controller is `ci/jenkins/docker-compose.yml`: stock Jenkins, no build
+tools. On Coolify, deploy it as a Docker Compose application from this
+repository (base directory `/ci/jenkins`), or paste it into a Compose service.
 
-1. **Build from this repo.**
+- **Java 25.** The file defaults to `jenkins/jenkins:lts-jdk25`; set
-   - For an Application service: provider GitHub, branch `march-2023-03-21`,
+  `JENKINS_IMAGE` to pin a version, e.g. `jenkins/jenkins:2.587-jdk25`.
-     build type Dockerfile, Dockerfile path `ci/jenkins/Dockerfile`, build
+  Jenkins supports Java 25 until September 2029, Java 21 only until September
-     context `ci/jenkins`.
+  2027, and Java 17 not at all. Keep the agent on the same Java major version;
-   - For a Compose service: change its `jenkins` service to `build:` from
+  its image defaults to Java 25.
-     `ci/jenkins` as in `ci/jenkins/docker-compose.yml`.
+- **`restart: unless-stopped`** is in the file: a restart from the Jenkins UI
-   - Either way, **keep the same volume on `/var/jenkins_home`**.
+  exits the container with code 5 and relies on Docker to start it again.
-2. **Match the Jenkins version.** Set the build argument `JENKINS_IMAGE` to the
+- **Turn on "Connect to predefined network"** (Advanced) so the agent can
-   image the service runs today (e.g. `jenkins/jenkins:2.492.3-lts-jdk17`).
+  reach Jenkins by its container name, `jenkins-<uuid>`.
-   The default is `jenkins/jenkins:lts-jdk17`.
+- **Run no builds on it:** **Manage Jenkins → Nodes → Built-In Node →
-3. **Add the environment:**
+  Configure**, set **Number of executors** to `0`.
+
+Earlier versions of this file built the controller from `ci/jenkins/Dockerfile`,
+which is now the agent's image. The service and data volume keep their names,
+so an existing Jenkins deployed from this file keeps its jobs and credentials
+when redeployed; it switches to the stock image, and its builds wait until
+the agent (step 3) is connected.
+
+That redeploy also upgrades Jenkins, and from Java 17 (the old default) to
+Java 25, unless `JENKINS_IMAGE` pins your current image. Before it:
+
+1. **Back up the data volume** (`jenkins_home`), e.g. with a Coolify volume
+   backup or `docker run --rm -v <volume>:/v -v "$PWD":/b alpine tar czf
+   /b/jenkins_home.tgz -C /v .`, so you can go back to the old image.
+2. After it starts, open **Manage Jenkins**. Plugins that need a newer
+   version show up there as failed to load; update them under **Plugins →
+   Updates** and restart.
+3. Run the job once (**LIST_SNAPSHOTS** is a quick check) before relying on it.
+
+### 2. The agent node
+
+**Manage Jenkins → Nodes → New Node**, a permanent agent:
+
+| Setting | Value |
+|---|---|
+| Name | `dorknet-builder` |
+| Labels | `dorknet-builder` |
+| Remote root directory | `/home/jenkins/agent` |
+| Launch method | Launch agent by connecting it to the controller, with **Use WebSocket** ticked |
+| Number of executors | 1 |
+
+Save, open the node, and copy its **secret**.
+
+### 3. Deploy the agent on Coolify
+
+1. **Create a Docker Compose application** from this repository: build pack
+   Docker Compose, base directory **`/ci/jenkins`**, compose location
+   **`/docker-compose.agent.yml`**. The base directory is the build context, so with
+   `/` the build would pick up the server's `Dockerfile` at the repo root. It
+   needs no domain. Under Advanced, turn on **Connect to predefined network**.
+2. **Set its environment:**
 
    | Variable | Value |
    |---|---|
+   | `JENKINS_URL` | Jenkins' **internal** address, `http://jenkins-<service uuid>:8080/`. Not the public URL, which is behind Cloudflare Access. |
+   | `JENKINS_SECRET` | the node secret from step 2 |
+   | `JENKINS_AGENT_NAME` | optional, default `dorknet-builder` |
    | `ARCHIVE_PC_REPO` | Steam repository folder on the FTP, e.g. `Rec Room Versions/steam-v2` (plain text, not URL-encoded) |
    | `ARCHIVE_QUEST_REPO` | Quest repository folder, e.g. `Rec Room Versions/quest-v2` |
    | `ARCHIVE_FTP_TLS` | `explicit` (default), `implicit` or `none` |
+   | `DOWNLOADS_S3_ENDPOINT` | the downloads bucket's S3 API, internal address, e.g. `http://garage-<service uuid>:3900` |
+   | `DOWNLOADS_BUCKET` | its name, e.g. `dorknet-downloads` |
 
-4. **Redeploy.** The first image build downloads the Android NDK (~1.5 GB) and
+3. **Deploy.** The first image build downloads the Android NDK (~1.5 GB) and
-   takes a while.
+   takes a while. The node shows as connected in Jenkins once the agent is up.
 
-### 2. Plugins
+Both Jenkins and the agent join the server's predefined Coolify network, so
+the agent reaches Jenkins by container name. Check the name under the Jenkins
+service's containers before using it.
+
+### 4. Plugins
 
 The job needs these plugins:
 
@@ -96,7 +164,7 @@ The job needs these plugins:
 
 Most are part of the suggested set.
 
-### 3. Credentials
+### 5. Credentials
 
 Add these under **Manage Jenkins → Credentials**, with exactly these IDs:
 
@@ -109,13 +177,14 @@ Add these under **Manage Jenkins → Credentials**, with exactly these IDs:
 | `quest-keystore-password` | Secret text | its store/key password |
 | `clientmod-config` | Secret file | your `dorknet-clientmod.json` |
 | `quest-config` | Secret file | your `dorknet-quest.cfg` |
+| `downloads-s3` | Username with password | S3 key for the downloads bucket: access key ID / secret. Give it read and write on that bucket only. |
 
 **Use the keystore your players' installs were signed with**, the
 `dorknet-debug.keystore` from your earlier local builds (password `dorknet`). A
 different key means every player has to uninstall before installing a CI
 build.
 
-### 4. The job
+### 6. The job
 
 1. Create a **New Item → Pipeline** (e.g. `dorknet-client`).
 2. Under **Pipeline**, choose **Pipeline script from SCM**:
@@ -126,7 +195,7 @@ build.
 4. Run it once with **Build** so Jenkins learns the parameters. Later runs
    offer **Build with Parameters**.
 
-### 5. GitHub webhook
+### 7. GitHub webhook
 
 In the repository's **Settings → Webhooks**:
 
@@ -134,7 +203,36 @@ In the repository's **Settings → Webhooks**:
 - Content type: `application/json`
 - Events: just the push event
 
+### 8. The downloads bucket
+
+`publish.py upload` puts the files and a `manifest.json` describing them in
+an S3 bucket. The DorkNet server reads that bucket over HTTP on the internal
+network and offers the files on its site (`docs/deploy.md` → Client
+downloads), so the bucket needs no public route. With Garage (the S3 store on
+the Coolify server):
+
+1. **Bucket and key.** Create the bucket, turn on Garage's website access for
+   it (the server reads it through the web endpoint, port `3902`, which picks
+   the bucket by the request's Host header), and give Jenkins a key for that
+   bucket only:
+
+   ```sh
+   garage bucket create dorknet-downloads
+   garage bucket website --allow dorknet-downloads
+   garage key create jenkins-downloads
+   garage bucket allow --read --write --key <key id> dorknet-downloads
+   ```
+
+   Store the key as the `downloads-s3` credential.
+2. **Server.** Point the DorkNet server at the web endpoint:
+   `DorkNet__Downloads__BaseUrl=http://garage-<service uuid>:3902` and
+   `DorkNet__Downloads__Host=dorknet-downloads`.
+
+Each release keeps one copy of each file; a new build overwrites it. The
+manifest is uploaded last, so the site only ever lists files that are fully
+uploaded.
+
 ### Disk
 
-The restored stock builds are cached in `jenkins_home/dorknet-cache`, several
+The agent's `dorknet_cache` volume holds the restored stock builds, several GB
-GB each. Jenkins keeps the artifacts of the last 3 runs.
+each. Jenkins keeps the artifacts of the last 3 runs, on its own volume.

ci/jenkins/docker-compose.agent.yml

@@ -0,0 +1,41 @@
+# DorkNet build agent on Coolify. Deploy as its own Docker Compose application
+# from this repository with base directory /ci/jenkins, compose location
+# /docker-compose.agent.yml and "Connect to predefined network" on. It needs no
+# domain: it dials out to Jenkins over WebSocket on Docker's internal network.
+# Set the variables in the resource's Environment Variables; see
+# ci/jenkins/README.md. The controller is docker-compose.yml.
+services:
+  agent:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: unless-stopped
+    # Reap the processes build tools leave behind (dotnet build servers,
+    # Gradle daemons, adb), as the inbound-agent image's docs ask for.
+    init: true
+    environment:
+      # Jenkins as seen from inside Docker (e.g. http://<jenkins container
+      # name>:8080), not the public URL, which is behind Cloudflare Access.
+      # Both are needed at runtime only; keep them out of the image build so
+      # the node secret never reaches a build argument.
+      JENKINS_URL: ${JENKINS_URL}
+      JENKINS_AGENT_NAME: ${JENKINS_AGENT_NAME:-dorknet-builder}
+      JENKINS_SECRET: ${JENKINS_SECRET}
+      JENKINS_WEB_SOCKET: "true"
+      JENKINS_AGENT_WORKDIR: /home/jenkins/agent
+      # Where the stock builds live in the archive (restic repository folders
+      # on the FTP, plain text). Overridable per run.
+      ARCHIVE_PC_REPO: ${ARCHIVE_PC_REPO:-}
+      ARCHIVE_QUEST_REPO: ${ARCHIVE_QUEST_REPO:-}
+      ARCHIVE_FTP_TLS: ${ARCHIVE_FTP_TLS:-explicit}
+      # The downloads bucket (S3 API, internal address) the builds are
+      # published to. See ci/downloads/publish.py.
+      DOWNLOADS_S3_ENDPOINT: ${DOWNLOADS_S3_ENDPOINT:-}
+      DOWNLOADS_BUCKET: ${DOWNLOADS_BUCKET:-}
+    volumes:
+      - dorknet_cache:/home/jenkins/dorknet-cache
+      - agent_work:/home/jenkins/agent
+
+volumes:
+  dorknet_cache:
+  agent_work:

ci/jenkins/docker-compose.yml

@@ -1,29 +1,23 @@
-# Reference for the Jenkins service on Dokploy, built from this directory's
+# The Jenkins controller: stock Jenkins on Java 25. It runs no builds; the
-# Dockerfile (the stock Jenkins image plus the client build toolchain).
+# client builds run on the agent (docker-compose.agent.yml). On Coolify,
+# deploy it as a Docker Compose application from this repository (base
+# directory /ci/jenkins) or paste it into a Compose service, with "Connect to
+# predefined network" on so the agent can reach it. See ci/jenkins/README.md.
 #
-# Use it on your EXISTING Jenkins service rather than as a second Jenkins.
+# Earlier versions of this file built the controller from ci/jenkins/Dockerfile,
-# The data volume must be the one Jenkins uses today, or Jenkins starts empty.
+# which is now the agent's image. The service name and data volume are the
-# Point `volumes:` at that volume (and its name under the top-level
+# same, so redeploying an existing Jenkins from this file keeps its jobs and
-# `volumes:`, marked `external: true`), or keep this service's name and volume
+# credentials; its builds then wait for the agent.
-# if they already match. See ci/jenkins/README.md.
 services:
   jenkins:
-    build:
+    # Pin a version with e.g. jenkins/jenkins:2.587-jdk25. Keep the agent on
-      context: .
+    # the same Java major version (its image defaults to Java 25).
-      dockerfile: Dockerfile
+    image: ${JENKINS_IMAGE:-jenkins/jenkins:lts-jdk25}
-      args:
+    # A restart from the Jenkins UI exits with code 5 and relies on Docker to
-        # The image your Jenkins runs today, so the Jenkins version and Java
+    # start it again.
-        # stay the same.
-        JENKINS_IMAGE: ${JENKINS_IMAGE:-jenkins/jenkins:lts-jdk17}
     restart: unless-stopped
     expose:
       - "8080"
-    environment:
-      # Where the stock builds live in the archive (restic repository folders
-      # on the FTP), as plain text. Overridable per run.
-      ARCHIVE_PC_REPO: ${ARCHIVE_PC_REPO:-}
-      ARCHIVE_QUEST_REPO: ${ARCHIVE_QUEST_REPO:-}
-      ARCHIVE_FTP_TLS: ${ARCHIVE_FTP_TLS:-explicit}
     volumes:
       - jenkins_home:/var/jenkins_home