diff --git a/DorkNet.Server/Startup/DatabaseBootstrap.cs b/DorkNet.Server/Startup/DatabaseBootstrap.cs index ae26d66..64ed35a 100644 --- a/DorkNet.Server/Startup/DatabaseBootstrap.cs +++ b/DorkNet.Server/Startup/DatabaseBootstrap.cs @@ -15,9 +15,9 @@ namespace DorkNet.Server.Startup; /// SQLite — Database.Migrate() after /// reconciles any legacy /// migration-history state. -/// Postgres — EnsureCreated behind a transaction-scoped -/// advisory lock so concurrent replica boots don't race on -/// CREATE TABLE. Connect retries cover the +/// Postgres — EnsureCreated, with the whole bootstrap +/// (schema and seeds) behind a session advisory lock so concurrent +/// replica boots take turns instead of racing. Connect retries cover the /// container-orchestrated boot window where Postgres DNS isn't /// resolvable yet. /// @@ -40,62 +40,110 @@ public static class DatabaseBootstrap await ConfigureSqliteForLocalConcurrencyAsync(db); BaselineExistingSchemaIfNeeded(db); db.Database.Migrate(); + await RunUpgradesAndSeedsAsync(app, scope, db); + return; } - else + + // Postgres path: schema generated from the EF Core model via + // EnsureCreated, then the same upgrades and seeds as SQLite. The + // whole bootstrap holds a session-level advisory lock, so replicas + // booting against one database take turns: the first creates the + // schema and seeds, the rest find everything in place. Holding it + // only around EnsureCreated left the seeders racing (duplicate keys + // on a fresh database). + // + // Lock key 0x444F524B (ascii "DORK") is hard-coded; it just needs + // to be the same int64 across replicas and not collide with + // anything else in the database. It is released in the finally + // below, and Postgres drops it if the replica's connection dies, so + // a crashed replica can't wedge the next boot. + var conn = db.Database.GetDbConnection(); + + // Retry initial Postgres connect — on container-orchestrated boots + // (Coolify, docker-compose) the Postgres service's DNS often isn't + // resolvable yet when this app starts, surfacing as SocketException + // errno=11 EAGAIN from Npgsql's Dns.GetHostEntryOrAddressesCore. + // Postgres itself can also be up-but-not-accepting-connections for + // a few seconds while it replays WAL. 15 attempts × ~2s backoff + // covers the typical 5-30s warmup window without looping forever + // on a permanently broken config. + var connectLogger = app.Services.GetRequiredService() + .CreateLogger("DorkNet.Server.Bootstrap"); + for (var attempt = 1; ; attempt++) { - // Postgres path: schema generated from the EF Core model via - // EnsureCreated. Wrapped in a transaction-scoped advisory lock - // so concurrent replicas booting against the same database - // don't race on CREATE TABLE — only the first replica to take - // the lock runs EnsureCreated, the rest see "schema already - // exists" and proceed. - // - // Lock key 0x444F524B (ascii "DORK") is hard-coded; it just - // needs to be the same int64 across replicas and not collide - // with anything else in the database. pg_advisory_xact_lock - // releases automatically on COMMIT, so a crashed replica can't - // wedge the next boot. - var conn = db.Database.GetDbConnection(); - - // Retry initial Postgres connect — on container-orchestrated boots - // (Coolify, docker-compose) the Postgres service's DNS often isn't - // resolvable yet when this app starts, surfacing as SocketException - // errno=11 EAGAIN from Npgsql's Dns.GetHostEntryOrAddressesCore. - // Postgres itself can also be up-but-not-accepting-connections for - // a few seconds while it replays WAL. 15 attempts × ~2s backoff - // covers the typical 5-30s warmup window without looping forever - // on a permanently broken config. - var connectLogger = app.Services.GetRequiredService() - .CreateLogger("DorkNet.Server.Bootstrap"); - for (var attempt = 1; ; attempt++) + try { await conn.OpenAsync(); break; } + catch (Exception ex) when (attempt < 15 && + (ex is System.Net.Sockets.SocketException + || ex is Npgsql.NpgsqlException + || ex is System.Data.Common.DbException)) { - try { await conn.OpenAsync(); break; } - catch (Exception ex) when (attempt < 15 && - (ex is System.Net.Sockets.SocketException - || ex is Npgsql.NpgsqlException - || ex is System.Data.Common.DbException)) - { - var delay = TimeSpan.FromSeconds(Math.Min(attempt, 5)); - connectLogger.LogWarning( - "[bootstrap] Postgres connect attempt {Attempt}/15 failed ({Type}: {Message}); retrying in {Delay}s", - attempt, ex.GetType().Name, ex.Message, delay.TotalSeconds); - await Task.Delay(delay); - } + var delay = TimeSpan.FromSeconds(Math.Min(attempt, 5)); + connectLogger.LogWarning( + "[bootstrap] Postgres connect attempt {Attempt}/15 failed ({Type}: {Message}); retrying in {Delay}s", + attempt, ex.GetType().Name, ex.Message, delay.TotalSeconds); + await Task.Delay(delay); } - // Begin the transaction through EF, not on the raw connection: - // since EF Core 10, EnsureCreated runs its commands through the - // migration executor, which starts its own transaction unless EF - // already has one, and fails on a fresh database with "A - // transaction is already in progress" when the open one is - // invisible to it. - await using var tx = await db.Database.BeginTransactionAsync(); - await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_xact_lock(1146246987);"); - db.Database.EnsureCreated(); - await tx.CommitAsync(); - - await conn.CloseAsync(); } + // The connection stays open until the finally, so every step below + // (they all share this scope's DbContext) runs on the session that + // holds the lock. Waiting for it has no command timeout: another + // replica may be seeding a fresh database (thumbnail downloads + // included) for well over the default 30 s. + connectLogger.LogInformation("[bootstrap] waiting for the database bootstrap lock"); + var previousTimeout = db.Database.GetCommandTimeout(); + db.Database.SetCommandTimeout(0); + try + { + await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_lock(1146246987);"); + } + finally + { + db.Database.SetCommandTimeout(previousTimeout); + } + connectLogger.LogInformation("[bootstrap] holding the database bootstrap lock"); + try + { + // Begin the transaction through EF, not on the raw connection: + // EnsureCreated runs its commands through the migration executor, + // which starts its own transaction unless EF already has one, and + // fails on a fresh database with "A transaction is already in + // progress" when the open one is invisible to it. + await using (var tx = await db.Database.BeginTransactionAsync()) + { + db.Database.EnsureCreated(); + await tx.CommitAsync(); + } + await RunUpgradesAndSeedsAsync(app, scope, db); + } + finally + { + // Explicit: Npgsql pools connections, so closing alone would not + // end the session that holds the lock. If the connection is + // already broken the unlock fails too; that must not replace the + // real bootstrap error, and the connection still gets closed + // (Postgres then drops the lock with the session). + try + { + await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_unlock(1146246987);"); + } + catch (Exception ex) + { + connectLogger.LogWarning(ex, "[bootstrap] releasing the database bootstrap lock failed"); + } + finally + { + await conn.CloseAsync(); + } + } + } + + /// The idempotent post-schema sequence both providers run: + /// column/table upgrades, system account and backfills, legacy data + /// transforms, canonical rooms and the per-feature seeders. Flips the + /// /healthz readiness flag at the end. + private static async Task RunUpgradesAndSeedsAsync(WebApplication app, IServiceScope scope, DorkNetDbContext db) + { // Signup-code + pending-device tables. These post-date the // consolidated Initial migration, and the Postgres path is // EnsureCreated-only (no migration replay), so neither provider diff --git a/Jenkinsfile b/Jenkinsfile index 6182520..ef53ee8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,6 +11,10 @@ // // On a push, only the platforms whose code changed are rebuilt. A manual run // picks the platforms with PLATFORMS. See ci/jenkins/README.md. +// +// Finished builds are renamed for players and published to the downloads bucket +// (ci/downloads) unless PUBLISH_DOWNLOADS is off. The build scripts keep their +// own file names, which the launcher depends on. // The builds this branch targets. Any other archived build can be picked per // run with the *_SNAPSHOT_TAGS parameters. @@ -18,6 +22,11 @@ def PC_VERSION_KEY = 'march_2023_03_21' def PC_DEFAULT_TAGS = 'manifest=7490748483298966814' // Steam 2023-03-21 (appVersion 20230317) def QUEST_DEFAULT_TAGS = 'version-code=2128210' // Quest 20230317 +// How this branch's builds are listed on the site's Downloads page. +def RELEASE_ID = '2023.03' +def RELEASE_TITLE = 'March 2023' +def RELEASE_GAME_VERSION = '2023-03-21' + def buildPc = false def buildQuest = false @@ -49,6 +58,8 @@ pipeline { description: 'Quest restic repository folder on the FTP. Empty: ARCHIVE_QUEST_REPO set on the agent.') booleanParam(name: 'LIST_SNAPSHOTS', defaultValue: false, description: 'Only list the snapshots matching the tags in both repositories (to find a build), then stop.') + booleanParam(name: 'PUBLISH_DOWNLOADS', defaultValue: true, + description: 'Publish what this run built to the downloads bucket (listed on the site Downloads page). Push builds publish too.') } stages { @@ -178,11 +189,35 @@ pipeline { } } } + + stage('Downloads') { + when { expression { buildPc || buildQuest } } + steps { + sh """ + python3 ci/downloads/publish.py stage --out out --release ${RELEASE_ID} \\ + --title '${RELEASE_TITLE}' --game-version ${RELEASE_GAME_VERSION} --version-key ${PC_VERSION_KEY} + """ + script { + if (params.PUBLISH_DOWNLOADS) { + // The bucket's S3 endpoint and name come from the agent's + // environment (DOWNLOADS_S3_ENDPOINT / DOWNLOADS_BUCKET). + withCredentials([usernamePassword(credentialsId: 'downloads-s3', + usernameVariable: 'DOWNLOADS_S3_KEY_ID', passwordVariable: 'DOWNLOADS_S3_SECRET')]) { + sh 'python3 ci/downloads/publish.py upload --staged out/downloads' + } + } else { + echo 'PUBLISH_DOWNLOADS is off: the files are only archived.' + } + } + } + } } post { success { - archiveArtifacts artifacts: 'out/**', allowEmptyArchive: true, fingerprint: true + // The renamed, player-facing files (and release.json): the same set the + // downloads page offers. + archiveArtifacts artifacts: 'out/downloads/**', allowEmptyArchive: true, fingerprint: true } cleanup { // Runs after archiving. Keeps the restic cache and restored stock diff --git a/ci/downloads/publish.py b/ci/downloads/publish.py new file mode 100644 index 0000000..7fe3d70 --- /dev/null +++ b/ci/downloads/publish.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Publish DorkNet client builds to the downloads bucket. + + stage the build output in out/ -> out/downloads// under the + names players see, plus release.json describing those files + upload that folder -> the downloads bucket, then merge release.json into + the bucket's manifest.json + +The DorkNet server reads the bucket and offers the files on its site (the +Downloads page, and Latest build on the Quest page); see docs/deploy.md. + +The build scripts keep their own file names on purpose: the launcher fetches +dorknet-clientpatch-.zip from GitHub Releases by that exact name. +Only the published copies are renamed: + + dorknet-client-.zip DorkNet--PC-Full.zip + dorknet-clientpatch-.zip DorkNet--PC-Patch.zip + DorkNet-Quest-/DorkNet-.xapk DorkNet--Quest.xapk + DorkNet-Quest-/install.*, README DorkNet--Quest-Installer.zip + +The server reads the bucket over plain HTTP on the internal network (e.g. +Garage's web endpoint); nothing needs to be public. upload talks to its S3 +API with rclone; the endpoint, bucket and key come from the environment: + + DOWNLOADS_S3_ENDPOINT S3 API URL, e.g. http://garage-:3900 + DOWNLOADS_BUCKET bucket name, e.g. dorknet-downloads + DOWNLOADS_S3_REGION optional, default garage + DOWNLOADS_S3_KEY_ID / DOWNLOADS_S3_SECRET + + python3 ci/downloads/publish.py stage --out out --release 2023.03 \\ + --title "March 2023" --game-version 2023-03-21 --version-key march_2023_03_21 + python3 ci/downloads/publish.py upload --staged out/downloads +""" +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import os +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import NoReturn + +NO_CACHE = "Cache-Control: no-cache" + +INSTALLER_NOTE = """Unzip these files into the same folder as {xapk}, then +run install.cmd (Windows) or ./install.sh (macOS / Linux). + +""" + + +def fail(msg: str) -> NoReturn: + print(f"error: {msg}", file=sys.stderr) + sys.exit(1) + + +def step(msg: str) -> None: + print(f"==> {msg}", flush=True) + + +def human(n: int) -> str: + return f"{n / 2**20:,.1f} MB" if n >= 2**20 else f"{max(1, round(n / 2**10))} KB" + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + while chunk := f.read(1 << 20): + h.update(chunk) + return h.hexdigest() + + +def link_or_copy(src: Path, dst: Path) -> None: + """Hard-link where possible: the full client is several GB.""" + dst.unlink(missing_ok=True) + try: + os.link(src, dst) + except OSError: + shutil.copy2(src, dst) + + +def game_exe(full_zip: Path) -> str | None: + """The game's exe: the one with a matching _Data folder at the root.""" + with zipfile.ZipFile(full_zip) as z: + names = z.namelist() + tops = {n.split("/", 1)[0] for n in names} + exes = sorted(t for t in tops if t.lower().endswith(".exe") and f"{t[:-4]}_Data" in tops) + return exes[0] if exes else None + + +# ---- stage ---------------------------------------------------------------- + +def cmd_stage(a: argparse.Namespace) -> None: + out = Path(a.out) + dest = Path(a.dest) if a.dest else out / "downloads" + rel_dir = dest / a.release + if rel_dir.exists(): + shutil.rmtree(rel_dir) + rel_dir.mkdir(parents=True) + prefix = f"DorkNet-{a.release}" + + staged: list[tuple[str, str, Path]] = [] # (id, platform, staged file) + extra: dict[str, str] = {} + + full = out / "pc" / f"dorknet-client-{a.version_key}.zip" + patch = out / "pc" / f"dorknet-clientpatch-{a.version_key}.zip" + if full.is_file(): + dst = rel_dir / f"{prefix}-PC-Full.zip" + link_or_copy(full, dst) + staged.append(("pc-full", "pc", dst)) + exe = game_exe(full) + if exe: + extra["pc_exe"] = exe + if patch.is_file(): + dst = rel_dir / f"{prefix}-PC-Patch.zip" + link_or_copy(patch, dst) + staged.append(("pc-patch", "pc", dst)) + + packages = sorted((out / "quest").glob("DorkNet-Quest-*/")) + if len(packages) > 1: + fail(f"more than one Quest package folder in {out / 'quest'}: {', '.join(p.name for p in packages)}") + if packages: + pkg = packages[0] + xapks = sorted(pkg.glob("*.xapk")) + if len(xapks) != 1: + fail(f"expected one .xapk in {pkg}, found {len(xapks)}") + xapk = rel_dir / f"{prefix}-Quest.xapk" + link_or_copy(xapks[0], xapk) + staged.append(("quest", "quest", xapk)) + + installer = rel_dir / f"{prefix}-Quest-Installer.zip" + with zipfile.ZipFile(installer, "w", zipfile.ZIP_DEFLATED) as z: + for name in ("install.cmd", "install.ps1", "install.sh"): + src = pkg / name + if not src.is_file(): + fail(f"{src} missing") + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.external_attr = (0o755 if name.endswith(".sh") else 0o644) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, src.read_bytes()) + readme = pkg / "README.txt" + text = INSTALLER_NOTE.format(xapk=xapk.name) + (readme.read_text(encoding="utf-8") if readme.is_file() else "") + info = zipfile.ZipInfo("README.txt", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, text) + staged.append(("quest-installer", "quest", installer)) + + if not staged: + fail(f"nothing to publish in {out}") + + built_at = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + files = [] + for fid, platform, path in staged: + step(f"{path.name}: hashing") + files.append({ + "id": fid, + "platform": platform, + "name": path.name, + "path": f"{a.release}/{path.name}", + "size": path.stat().st_size, + "sha256": sha256(path), + "build": a.build, + "commit": a.commit, + "built_at": built_at, + }) + release = {"id": a.release, "title": a.title, "game_version": a.game_version, **extra, "files": files} + (dest / "release.json").write_text(json.dumps(release, indent=2) + "\n", encoding="utf-8") + print() + for f in files: + print(f" {f['name']} {human(f['size'])} {f['sha256'][:16]}…") + print(f"Staged {len(files)} file(s) in {rel_dir}") + + +# ---- upload --------------------------------------------------------------- + +def rclone_env() -> dict[str, str]: + env = dict(os.environ) + for k in ("DOWNLOADS_S3_ENDPOINT", "DOWNLOADS_BUCKET", "DOWNLOADS_S3_KEY_ID", "DOWNLOADS_S3_SECRET"): + if not env.get(k): + fail(f"{k} is not set") + env.update({ + "RCLONE_CONFIG_DL_TYPE": "s3", + "RCLONE_CONFIG_DL_PROVIDER": "Other", + "RCLONE_CONFIG_DL_ENDPOINT": env["DOWNLOADS_S3_ENDPOINT"], + "RCLONE_CONFIG_DL_REGION": env.get("DOWNLOADS_S3_REGION") or "garage", + "RCLONE_CONFIG_DL_ACCESS_KEY_ID": env["DOWNLOADS_S3_KEY_ID"], + "RCLONE_CONFIG_DL_SECRET_ACCESS_KEY": env["DOWNLOADS_S3_SECRET"], + "RCLONE_CONFIG_DL_FORCE_PATH_STYLE": "true", + "RCLONE_CONFIG_DL_NO_CHECK_BUCKET": "true", + }) + return env + + +def rclone(env: dict[str, str], *args: str, stdin: str | None = None, check: bool = True) -> subprocess.CompletedProcess: + r = subprocess.run(["rclone", *args], env=env, input=stdin, capture_output=True, text=True) + if check and r.returncode != 0: + fail(f"rclone {args[0]} failed ({r.returncode}): {r.stderr.strip()}") + return r + + +def merge(manifest: dict, release: dict) -> dict: + """Replace this release's files that were just built; keep the others + (a push build may rebuild only one platform).""" + releases = [r for r in manifest.get("releases", []) if r.get("id") != release["id"]] + old = next((r for r in manifest.get("releases", []) if r.get("id") == release["id"]), {}) + new_ids = {f["id"] for f in release["files"]} + kept = [f for f in old.get("files", []) if f.get("id") not in new_ids] + merged = {**old, **release, "files": kept + release["files"]} + if "pc_exe" not in release and "pc_exe" in old: + merged["pc_exe"] = old["pc_exe"] + order = ["pc-full", "pc-patch", "quest", "quest-installer"] + merged["files"].sort(key=lambda f: order.index(f["id"]) if f["id"] in order else len(order)) + releases.append(merged) + releases.sort(key=lambda r: r["id"], reverse=True) + out = {**manifest, "releases": releases, + "updated": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()} + return out + + +def cmd_upload(a: argparse.Namespace) -> None: + staged = Path(a.staged) + release = json.loads((staged / "release.json").read_text(encoding="utf-8")) + env = rclone_env() + bucket = f"dl:{env['DOWNLOADS_BUCKET']}" + rel = release["id"] + + # 1. the files, before the manifest that points at them + for f in release["files"]: + step(f"uploading {f['name']} ({human(f['size'])})") + rclone(env, "copyto", str(staged / f["path"]), f"{bucket}/{f['path']}", + "--header-upload", NO_CACHE, "--s3-upload-concurrency", "8", "--s3-chunk-size", "64M") + + # 2. merge into the live manifest + r = rclone(env, "cat", f"{bucket}/manifest.json", check=False) + manifest = json.loads(r.stdout) if r.returncode == 0 and r.stdout.strip() else {} + manifest = merge(manifest, release) + + # 3. the manifest, last: it is what makes the new files visible + step("uploading manifest.json") + rclone(env, "rcat", f"{bucket}/manifest.json", "--header-upload", NO_CACHE, + "--header-upload", "Content-Type: application/json", stdin=json.dumps(manifest, indent=2) + "\n") + + # 4. drop files of this release the manifest no longer lists (renames) + listed = {f["path"] for r in manifest["releases"] if r["id"] == rel for f in r["files"]} + r = rclone(env, "lsf", f"{bucket}/{rel}/", "--files-only", check=False) + for name in r.stdout.split(): + if f"{rel}/{name}" not in listed: + step(f"removing stale {rel}/{name}") + rclone(env, "deletefile", f"{bucket}/{rel}/{name}") + print(f"Published {len(release['files'])} file(s) for {release['title']}.") + + +# ---- cli ------------------------------------------------------------------ + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest="cmd", required=True) + + s = sub.add_parser("stage", help="out/ -> out/downloads// + release.json") + s.add_argument("--out", default="out", help="the build output folder (default: out)") + s.add_argument("--dest", help="where to stage (default: /downloads)") + s.add_argument("--release", required=True, help="release id used in file names and paths, e.g. 2023.03") + s.add_argument("--title", required=True, help='shown on the Downloads page, e.g. "March 2023"') + s.add_argument("--game-version", required=True, help="the stock build's date, e.g. 2023-03-21") + s.add_argument("--version-key", required=True, help="the PC build's version key, e.g. march_2023_03_21") + s.add_argument("--build", default=os.environ.get("BUILD_NUMBER", ""), help="CI build number (default: $BUILD_NUMBER)") + s.add_argument("--commit", default=os.environ.get("GIT_COMMIT", ""), help="source commit (default: $GIT_COMMIT)") + s.set_defaults(fn=cmd_stage) + + u = sub.add_parser("upload", help="staged folder -> downloads bucket + manifest.json") + u.add_argument("--staged", default="out/downloads", help="the folder `stage` wrote (default: out/downloads)") + u.set_defaults(fn=cmd_upload) + + a = p.parse_args() + a.fn(a) + + +if __name__ == "__main__": + main() diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index 2b6b8e3..0540fec 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -11,14 +11,26 @@ The builds run on a **build agent**, not on Jenkins itself: job runs on it through the `dorknet-builder` label. - **Jenkins itself:** stays on the stock image and runs no builds. -| Platform | Output (Jenkins artifacts) | How | -|---|---|---| -| PC | `out/pc/dorknet-clientpatch-march_2023_03_21.zip`: MelonLoader, mod, config template, Steam stub, `manifest.json`; no game files | `tools/build_pc.py patcher` | -| PC | `out/pc/dorknet-client-march_2023_03_21.zip`: the stock Steam build with the patcher applied and your config; ready to run | `tools/build_pc.py client` | -| Quest | `out/quest/dorknet-recroom-.apk` and `out/quest/DorkNet-Quest-/` (`.xapk` + installers) | `DorkNet.Quest/scripts/build.py build` | +Each build is published to the **downloads bucket** under a name players can +read, and the same files are kept as Jenkins artifacts +(`out/downloads/2023.03/`). The DorkNet server offers them on its site: the +**Downloads** page, and **Latest build** on the **Quest** page +(`docs/deploy.md` → Client downloads). -The full PC client and the Quest packages contain game files, so keep them on -Jenkins. Don't publish them anywhere public. +| Download | What it is | Built from | +|---|---|---| +| `DorkNet-2023.03-PC-Full.zip` | the stock Steam build with the patcher applied and your config; ready to run | `tools/build_pc.py client` → `dorknet-client-march_2023_03_21.zip` | +| `DorkNet-2023.03-PC-Patch.zip` | MelonLoader, mod, config template, Steam stub, `manifest.json`; no game files | `tools/build_pc.py patcher` → `dorknet-clientpatch-march_2023_03_21.zip` | +| `DorkNet-2023.03-Quest.xapk` | patched APK + OBB, for the browser installer or SideQuest | `DorkNet.Quest/scripts/build.py build` → `DorkNet-Quest-/DorkNet-.xapk` | +| `DorkNet-2023.03-Quest-Installer.zip` | `install.cmd` / `install.ps1` / `install.sh` + README, unzipped next to the `.xapk` | the same package folder | + +The build scripts keep their own names: the launcher downloads +`dorknet-clientpatch-.zip` from GitHub Releases by that exact +name. `ci/downloads/publish.py` renames only the published copies. + +The full PC client and the Quest packages contain game files. Keep the bucket +internal and the site behind an access proxy; don't publish them anywhere +public. ## Where the stock builds come from @@ -54,7 +66,11 @@ Resolve a table for another build locally with `build.py build --dump-cs`. - `Jenkinsfile` and `ci/jenkins/` rebuild both. - **Build with Parameters** runs a manual build: - `PLATFORMS` (both / pc / quest), and `PC_FULL_CLIENT` for the full PC client; - - snapshot tags, and per-run repository overrides. + - snapshot tags, and per-run repository overrides; + - `PUBLISH_DOWNLOADS` (on by default) to publish to the downloads bucket. + +Push builds publish too, but only the platform they rebuilt: the manifest +keeps the other platform's last files. ## Setup @@ -124,6 +140,8 @@ Save, open the node, and copy its **secret**. | `ARCHIVE_PC_REPO` | Steam repository folder on the FTP, e.g. `Rec Room Versions/steam-v2` (plain text, not URL-encoded) | | `ARCHIVE_QUEST_REPO` | Quest repository folder, e.g. `Rec Room Versions/quest-v2` | | `ARCHIVE_FTP_TLS` | `explicit` (default), `implicit` or `none` | + | `DOWNLOADS_S3_ENDPOINT` | the downloads bucket's S3 API, internal address, e.g. `http://garage-:3900` | + | `DOWNLOADS_BUCKET` | its name, e.g. `dorknet-downloads` | 3. **Deploy.** The first image build downloads the Android NDK (~1.5 GB) and takes a while. The node shows as connected in Jenkins once the agent is up. @@ -159,6 +177,7 @@ Add these under **Manage Jenkins → Credentials**, with exactly these IDs: | `quest-keystore-password` | Secret text | its store/key password | | `clientmod-config` | Secret file | your `dorknet-clientmod.json` | | `quest-config` | Secret file | your `dorknet-quest.cfg` | +| `downloads-s3` | Username with password | S3 key for the downloads bucket: access key ID / secret. Give it read and write on that bucket only. | **Use the keystore your players' installs were signed with**, the `dorknet-debug.keystore` from your earlier local builds (password `dorknet`). A @@ -184,6 +203,35 @@ In the repository's **Settings → Webhooks**: - Content type: `application/json` - Events: just the push event +### 8. The downloads bucket + +`publish.py upload` puts the files and a `manifest.json` describing them in +an S3 bucket. The DorkNet server reads that bucket over HTTP on the internal +network and offers the files on its site (`docs/deploy.md` → Client +downloads), so the bucket needs no public route. With Garage (the S3 store on +the Coolify server): + +1. **Bucket and key.** Create the bucket, turn on Garage's website access for + it (the server reads it through the web endpoint, port `3902`, which picks + the bucket by the request's Host header), and give Jenkins a key for that + bucket only: + + ```sh + garage bucket create dorknet-downloads + garage bucket website --allow dorknet-downloads + garage key create jenkins-downloads + garage bucket allow --read --write --key dorknet-downloads + ``` + + Store the key as the `downloads-s3` credential. +2. **Server.** Point the DorkNet server at the web endpoint: + `DorkNet__Downloads__BaseUrl=http://garage-:3902` and + `DorkNet__Downloads__Host=dorknet-downloads`. + +Each release keeps one copy of each file; a new build overwrites it. The +manifest is uploaded last, so the site only ever lists files that are fully +uploaded. + ### Disk The agent's `dorknet_cache` volume holds the restored stock builds, several GB diff --git a/ci/jenkins/docker-compose.agent.yml b/ci/jenkins/docker-compose.agent.yml index 1c9045c..86efec3 100644 --- a/ci/jenkins/docker-compose.agent.yml +++ b/ci/jenkins/docker-compose.agent.yml @@ -28,6 +28,10 @@ services: ARCHIVE_PC_REPO: ${ARCHIVE_PC_REPO:-} ARCHIVE_QUEST_REPO: ${ARCHIVE_QUEST_REPO:-} ARCHIVE_FTP_TLS: ${ARCHIVE_FTP_TLS:-explicit} + # The downloads bucket (S3 API, internal address) the builds are + # published to. See ci/downloads/publish.py. + DOWNLOADS_S3_ENDPOINT: ${DOWNLOADS_S3_ENDPOINT:-} + DOWNLOADS_BUCKET: ${DOWNLOADS_BUCKET:-} volumes: - dorknet_cache:/home/jenkins/dorknet-cache - agent_work:/home/jenkins/agent