From 6e5b011059b594da6d4dfbb5e17fe94ef6efbdbb Mon Sep 17 00:00:00 2001 From: Alexa <53471801+Alexa-RR@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:40:46 +0200 Subject: [PATCH 1/2] ci: publish client builds to a downloads page under player-facing names Each run now stages what it built as DorkNet--PC-Full.zip, -PC-Patch.zip, -Quest.xapk and -Quest-Installer.zip, writes their sizes and SHA-256 to a manifest, and uploads them with a static page to an S3 bucket served read-only (ci/downloads). The build scripts keep their own names, which the launcher depends on; only the published copies are renamed. PUBLISH_DOWNLOADS (on by default) turns the upload off per run. Jenkins archives the same renamed files instead of the raw build output. --- Jenkinsfile | 37 +++- ci/downloads/index.html | 243 +++++++++++++++++++++++ ci/downloads/publish.py | 294 ++++++++++++++++++++++++++++ ci/jenkins/README.md | 78 +++++++- ci/jenkins/docker-compose.agent.yml | 5 + 5 files changed, 648 insertions(+), 9 deletions(-) create mode 100644 ci/downloads/index.html create mode 100644 ci/downloads/publish.py diff --git a/Jenkinsfile b/Jenkinsfile index 6182520..b18f85b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,6 +11,10 @@ // // On a push, only the platforms whose code changed are rebuilt. A manual run // picks the platforms with PLATFORMS. See ci/jenkins/README.md. +// +// Finished builds are renamed for players and published to the downloads page +// (ci/downloads) unless PUBLISH_DOWNLOADS is off. The build scripts keep their +// own file names, which the launcher depends on. // The builds this branch targets. Any other archived build can be picked per // run with the *_SNAPSHOT_TAGS parameters. @@ -18,6 +22,11 @@ def PC_VERSION_KEY = 'march_2023_03_21' def PC_DEFAULT_TAGS = 'manifest=7490748483298966814' // Steam 2023-03-21 (appVersion 20230317) def QUEST_DEFAULT_TAGS = 'version-code=2128210' // Quest 20230317 +// How this branch's builds are listed on the downloads page. +def RELEASE_ID = '2023.03' +def RELEASE_TITLE = 'March 2023' +def RELEASE_GAME_VERSION = '2023-03-21' + def buildPc = false def buildQuest = false @@ -49,6 +58,8 @@ pipeline { description: 'Quest restic repository folder on the FTP. Empty: ARCHIVE_QUEST_REPO set on the agent.') booleanParam(name: 'LIST_SNAPSHOTS', defaultValue: false, description: 'Only list the snapshots matching the tags in both repositories (to find a build), then stop.') + booleanParam(name: 'PUBLISH_DOWNLOADS', defaultValue: true, + description: 'Publish what this run built to the downloads page. Push builds publish too.') } stages { @@ -178,11 +189,35 @@ pipeline { } } } + + stage('Downloads') { + when { expression { buildPc || buildQuest } } + steps { + sh """ + python3 ci/downloads/publish.py stage --out out --release ${RELEASE_ID} \\ + --title '${RELEASE_TITLE}' --game-version ${RELEASE_GAME_VERSION} --version-key ${PC_VERSION_KEY} + """ + script { + if (params.PUBLISH_DOWNLOADS) { + // The bucket's S3 endpoint and name come from the agent's + // environment (DOWNLOADS_S3_ENDPOINT / DOWNLOADS_BUCKET). + withCredentials([usernamePassword(credentialsId: 'downloads-s3', + usernameVariable: 'DOWNLOADS_S3_KEY_ID', passwordVariable: 'DOWNLOADS_S3_SECRET')]) { + sh 'python3 ci/downloads/publish.py upload --staged out/downloads' + } + } else { + echo 'PUBLISH_DOWNLOADS is off: the files are only archived.' + } + } + } + } } post { success { - archiveArtifacts artifacts: 'out/**', allowEmptyArchive: true, fingerprint: true + // The renamed, player-facing files (and release.json): the same set the + // downloads page offers. + archiveArtifacts artifacts: 'out/downloads/**', allowEmptyArchive: true, fingerprint: true } cleanup { // Runs after archiving. Keeps the restic cache and restored stock diff --git a/ci/downloads/index.html b/ci/downloads/index.html new file mode 100644 index 0000000..e5bec93 --- /dev/null +++ b/ci/downloads/index.html @@ -0,0 +1,243 @@ + + + + + + +DorkNet Downloads + + + + + + + + +
+
+ +
DorkNet Downloads
+
+ Sign out +
+
+ +
+
+

Get Rec Room for DorkNet

+

Pick your platform. Every file is built from the DorkNet source by our build server; the + SHA-256 next to each one lets you check your download.

+
+

Loading builds…

+
+

These builds contain Rec Room's own game files. They're for the DorkNet community only: please don't + re-share them.

+
+
+ + + + diff --git a/ci/downloads/publish.py b/ci/downloads/publish.py new file mode 100644 index 0000000..6812ebd --- /dev/null +++ b/ci/downloads/publish.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Publish DorkNet client builds to the downloads page. + + stage the build output in out/ -> out/downloads// under the + names players see, plus release.json describing those files + upload that folder -> the downloads bucket, then merge release.json into + the bucket's manifest.json and upload the page (index.html) + +The build scripts keep their own file names on purpose: the launcher fetches +dorknet-clientpatch-.zip from GitHub Releases by that exact name. +Only the published copies are renamed: + + dorknet-client-.zip DorkNet--PC-Full.zip + dorknet-clientpatch-.zip DorkNet--PC-Patch.zip + DorkNet-Quest-/DorkNet-.xapk DorkNet--Quest.xapk + DorkNet-Quest-/install.*, README DorkNet--Quest-Installer.zip + +The bucket is served read-only as a static website (Garage's web endpoint). +upload talks to its S3 API with rclone; the endpoint, bucket and key come +from the environment: + + DOWNLOADS_S3_ENDPOINT S3 API URL, e.g. http://garage-:3900 + DOWNLOADS_BUCKET bucket name, e.g. downloads.example.com + DOWNLOADS_S3_REGION optional, default garage + DOWNLOADS_S3_KEY_ID / DOWNLOADS_S3_SECRET + DORKNET_SITE_URL optional: the DorkNet server's website, linked + from the page for the browser Quest installer + + python3 ci/downloads/publish.py stage --out out --release 2023.03 \\ + --title "March 2023" --game-version 2023-03-21 --version-key march_2023_03_21 + python3 ci/downloads/publish.py upload --staged out/downloads +""" +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import os +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import NoReturn + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +PAGE_FILES = { # bucket path -> source + "index.html": HERE / "index.html", + "dorknet.svg": REPO / "DorkNet.Server" / "site" / "public" / "brand" / "dorknet.svg", +} +NO_CACHE = "Cache-Control: no-cache" + +INSTALLER_NOTE = """Unzip these files into the same folder as {xapk}, then +run install.cmd (Windows) or ./install.sh (macOS / Linux). + +""" + + +def fail(msg: str) -> NoReturn: + print(f"error: {msg}", file=sys.stderr) + sys.exit(1) + + +def step(msg: str) -> None: + print(f"==> {msg}", flush=True) + + +def human(n: int) -> str: + return f"{n / 2**20:,.1f} MB" if n >= 2**20 else f"{max(1, round(n / 2**10))} KB" + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + while chunk := f.read(1 << 20): + h.update(chunk) + return h.hexdigest() + + +def link_or_copy(src: Path, dst: Path) -> None: + """Hard-link where possible: the full client is several GB.""" + dst.unlink(missing_ok=True) + try: + os.link(src, dst) + except OSError: + shutil.copy2(src, dst) + + +def game_exe(full_zip: Path) -> str | None: + """The game's exe: the one with a matching _Data folder at the root.""" + with zipfile.ZipFile(full_zip) as z: + names = z.namelist() + tops = {n.split("/", 1)[0] for n in names} + exes = sorted(t for t in tops if t.lower().endswith(".exe") and f"{t[:-4]}_Data" in tops) + return exes[0] if exes else None + + +# ---- stage ---------------------------------------------------------------- + +def cmd_stage(a: argparse.Namespace) -> None: + out = Path(a.out) + dest = Path(a.dest) if a.dest else out / "downloads" + rel_dir = dest / a.release + if rel_dir.exists(): + shutil.rmtree(rel_dir) + rel_dir.mkdir(parents=True) + prefix = f"DorkNet-{a.release}" + + staged: list[tuple[str, str, Path]] = [] # (id, platform, staged file) + extra: dict[str, str] = {} + + full = out / "pc" / f"dorknet-client-{a.version_key}.zip" + patch = out / "pc" / f"dorknet-clientpatch-{a.version_key}.zip" + if full.is_file(): + dst = rel_dir / f"{prefix}-PC-Full.zip" + link_or_copy(full, dst) + staged.append(("pc-full", "pc", dst)) + exe = game_exe(full) + if exe: + extra["pc_exe"] = exe + if patch.is_file(): + dst = rel_dir / f"{prefix}-PC-Patch.zip" + link_or_copy(patch, dst) + staged.append(("pc-patch", "pc", dst)) + + packages = sorted((out / "quest").glob("DorkNet-Quest-*/")) + if len(packages) > 1: + fail(f"more than one Quest package folder in {out / 'quest'}: {', '.join(p.name for p in packages)}") + if packages: + pkg = packages[0] + xapks = sorted(pkg.glob("*.xapk")) + if len(xapks) != 1: + fail(f"expected one .xapk in {pkg}, found {len(xapks)}") + xapk = rel_dir / f"{prefix}-Quest.xapk" + link_or_copy(xapks[0], xapk) + staged.append(("quest", "quest", xapk)) + + installer = rel_dir / f"{prefix}-Quest-Installer.zip" + with zipfile.ZipFile(installer, "w", zipfile.ZIP_DEFLATED) as z: + for name in ("install.cmd", "install.ps1", "install.sh"): + src = pkg / name + if not src.is_file(): + fail(f"{src} missing") + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.external_attr = (0o755 if name.endswith(".sh") else 0o644) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, src.read_bytes()) + readme = pkg / "README.txt" + text = INSTALLER_NOTE.format(xapk=xapk.name) + (readme.read_text(encoding="utf-8") if readme.is_file() else "") + info = zipfile.ZipInfo("README.txt", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, text) + staged.append(("quest-installer", "quest", installer)) + + if not staged: + fail(f"nothing to publish in {out}") + + built_at = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + files = [] + for fid, platform, path in staged: + step(f"{path.name}: hashing") + files.append({ + "id": fid, + "platform": platform, + "name": path.name, + "path": f"{a.release}/{path.name}", + "size": path.stat().st_size, + "sha256": sha256(path), + "build": a.build, + "commit": a.commit, + "built_at": built_at, + }) + release = {"id": a.release, "title": a.title, "game_version": a.game_version, **extra, "files": files} + (dest / "release.json").write_text(json.dumps(release, indent=2) + "\n", encoding="utf-8") + print() + for f in files: + print(f" {f['name']} {human(f['size'])} {f['sha256'][:16]}…") + print(f"Staged {len(files)} file(s) in {rel_dir}") + + +# ---- upload --------------------------------------------------------------- + +def rclone_env() -> dict[str, str]: + env = dict(os.environ) + for k in ("DOWNLOADS_S3_ENDPOINT", "DOWNLOADS_BUCKET", "DOWNLOADS_S3_KEY_ID", "DOWNLOADS_S3_SECRET"): + if not env.get(k): + fail(f"{k} is not set") + env.update({ + "RCLONE_CONFIG_DL_TYPE": "s3", + "RCLONE_CONFIG_DL_PROVIDER": "Other", + "RCLONE_CONFIG_DL_ENDPOINT": env["DOWNLOADS_S3_ENDPOINT"], + "RCLONE_CONFIG_DL_REGION": env.get("DOWNLOADS_S3_REGION") or "garage", + "RCLONE_CONFIG_DL_ACCESS_KEY_ID": env["DOWNLOADS_S3_KEY_ID"], + "RCLONE_CONFIG_DL_SECRET_ACCESS_KEY": env["DOWNLOADS_S3_SECRET"], + "RCLONE_CONFIG_DL_FORCE_PATH_STYLE": "true", + "RCLONE_CONFIG_DL_NO_CHECK_BUCKET": "true", + }) + return env + + +def rclone(env: dict[str, str], *args: str, stdin: str | None = None, check: bool = True) -> subprocess.CompletedProcess: + r = subprocess.run(["rclone", *args], env=env, input=stdin, capture_output=True, text=True) + if check and r.returncode != 0: + fail(f"rclone {args[0]} failed ({r.returncode}): {r.stderr.strip()}") + return r + + +def merge(manifest: dict, release: dict) -> dict: + """Replace this release's files that were just built; keep the others + (a push build may rebuild only one platform).""" + releases = [r for r in manifest.get("releases", []) if r.get("id") != release["id"]] + old = next((r for r in manifest.get("releases", []) if r.get("id") == release["id"]), {}) + new_ids = {f["id"] for f in release["files"]} + kept = [f for f in old.get("files", []) if f.get("id") not in new_ids] + merged = {**old, **release, "files": kept + release["files"]} + if "pc_exe" not in release and "pc_exe" in old: + merged["pc_exe"] = old["pc_exe"] + order = ["pc-full", "pc-patch", "quest", "quest-installer"] + merged["files"].sort(key=lambda f: order.index(f["id"]) if f["id"] in order else len(order)) + releases.append(merged) + releases.sort(key=lambda r: r["id"], reverse=True) + out = {**manifest, "releases": releases, + "updated": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()} + site = os.environ.get("DORKNET_SITE_URL", "").rstrip("/") + if site: + out["site_url"] = site + return out + + +def cmd_upload(a: argparse.Namespace) -> None: + staged = Path(a.staged) + release = json.loads((staged / "release.json").read_text(encoding="utf-8")) + env = rclone_env() + bucket = f"dl:{env['DOWNLOADS_BUCKET']}" + rel = release["id"] + + # 1. the files, before the manifest that points at them + for f in release["files"]: + step(f"uploading {f['name']} ({human(f['size'])})") + rclone(env, "copyto", str(staged / f["path"]), f"{bucket}/{f['path']}", + "--header-upload", NO_CACHE, "--s3-upload-concurrency", "8", "--s3-chunk-size", "64M") + + # 2. merge into the live manifest + r = rclone(env, "cat", f"{bucket}/manifest.json", check=False) + manifest = json.loads(r.stdout) if r.returncode == 0 and r.stdout.strip() else {} + manifest = merge(manifest, release) + + # 3. the page, then the manifest + for path, src in PAGE_FILES.items(): + step(f"uploading {path}") + rclone(env, "copyto", str(src), f"{bucket}/{path}", "--header-upload", NO_CACHE) + step("uploading manifest.json") + rclone(env, "rcat", f"{bucket}/manifest.json", "--header-upload", NO_CACHE, + "--header-upload", "Content-Type: application/json", stdin=json.dumps(manifest, indent=2) + "\n") + + # 4. drop files of this release the manifest no longer lists (renames) + listed = {f["path"] for r in manifest["releases"] if r["id"] == rel for f in r["files"]} + r = rclone(env, "lsf", f"{bucket}/{rel}/", "--files-only", check=False) + for name in r.stdout.split(): + if f"{rel}/{name}" not in listed: + step(f"removing stale {rel}/{name}") + rclone(env, "deletefile", f"{bucket}/{rel}/{name}") + print(f"Published {len(release['files'])} file(s) for {release['title']}.") + + +# ---- cli ------------------------------------------------------------------ + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest="cmd", required=True) + + s = sub.add_parser("stage", help="out/ -> out/downloads// + release.json") + s.add_argument("--out", default="out", help="the build output folder (default: out)") + s.add_argument("--dest", help="where to stage (default: /downloads)") + s.add_argument("--release", required=True, help="release id used in file names and paths, e.g. 2023.03") + s.add_argument("--title", required=True, help='shown on the page, e.g. "March 2023"') + s.add_argument("--game-version", required=True, help="the stock build's date, e.g. 2023-03-21") + s.add_argument("--version-key", required=True, help="the PC build's version key, e.g. march_2023_03_21") + s.add_argument("--build", default=os.environ.get("BUILD_NUMBER", ""), help="CI build number (default: $BUILD_NUMBER)") + s.add_argument("--commit", default=os.environ.get("GIT_COMMIT", ""), help="source commit (default: $GIT_COMMIT)") + s.set_defaults(fn=cmd_stage) + + u = sub.add_parser("upload", help="staged folder -> downloads bucket + manifest.json + page") + u.add_argument("--staged", default="out/downloads", help="the folder `stage` wrote (default: out/downloads)") + u.set_defaults(fn=cmd_upload) + + a = p.parse_args() + a.fn(a) + + +if __name__ == "__main__": + main() diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index 2b6b8e3..b6ac455 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -11,14 +11,24 @@ The builds run on a **build agent**, not on Jenkins itself: job runs on it through the `dorknet-builder` label. - **Jenkins itself:** stays on the stock image and runs no builds. -| Platform | Output (Jenkins artifacts) | How | -|---|---|---| -| PC | `out/pc/dorknet-clientpatch-march_2023_03_21.zip`: MelonLoader, mod, config template, Steam stub, `manifest.json`; no game files | `tools/build_pc.py patcher` | -| PC | `out/pc/dorknet-client-march_2023_03_21.zip`: the stock Steam build with the patcher applied and your config; ready to run | `tools/build_pc.py client` | -| Quest | `out/quest/dorknet-recroom-.apk` and `out/quest/DorkNet-Quest-/` (`.xapk` + installers) | `DorkNet.Quest/scripts/build.py build` | +Each build is published to the **downloads page** under a name players can +read, and the same files are kept as Jenkins artifacts +(`out/downloads/2023.03/`): -The full PC client and the Quest packages contain game files, so keep them on -Jenkins. Don't publish them anywhere public. +| Download | What it is | Built from | +|---|---|---| +| `DorkNet-2023.03-PC-Full.zip` | the stock Steam build with the patcher applied and your config; ready to run | `tools/build_pc.py client` → `dorknet-client-march_2023_03_21.zip` | +| `DorkNet-2023.03-PC-Patch.zip` | MelonLoader, mod, config template, Steam stub, `manifest.json`; no game files | `tools/build_pc.py patcher` → `dorknet-clientpatch-march_2023_03_21.zip` | +| `DorkNet-2023.03-Quest.xapk` | patched APK + OBB, for the browser installer or SideQuest | `DorkNet.Quest/scripts/build.py build` → `DorkNet-Quest-/DorkNet-.xapk` | +| `DorkNet-2023.03-Quest-Installer.zip` | `install.cmd` / `install.ps1` / `install.sh` + README, unzipped next to the `.xapk` | the same package folder | + +The build scripts keep their own names: the launcher downloads +`dorknet-clientpatch-.zip` from GitHub Releases by that exact +name. `ci/downloads/publish.py` renames only the published copies. + +The full PC client and the Quest packages contain game files. The downloads +page is behind Cloudflare Access for that reason; don't publish them +anywhere public. ## Where the stock builds come from @@ -54,7 +64,11 @@ Resolve a table for another build locally with `build.py build --dump-cs`. - `Jenkinsfile` and `ci/jenkins/` rebuild both. - **Build with Parameters** runs a manual build: - `PLATFORMS` (both / pc / quest), and `PC_FULL_CLIENT` for the full PC client; - - snapshot tags, and per-run repository overrides. + - snapshot tags, and per-run repository overrides; + - `PUBLISH_DOWNLOADS` (on by default) to publish to the downloads page. + +Push builds publish too, but only the platform they rebuilt: the page keeps +the other platform's last files. ## Setup @@ -124,6 +138,9 @@ Save, open the node, and copy its **secret**. | `ARCHIVE_PC_REPO` | Steam repository folder on the FTP, e.g. `Rec Room Versions/steam-v2` (plain text, not URL-encoded) | | `ARCHIVE_QUEST_REPO` | Quest repository folder, e.g. `Rec Room Versions/quest-v2` | | `ARCHIVE_FTP_TLS` | `explicit` (default), `implicit` or `none` | + | `DOWNLOADS_S3_ENDPOINT` | the downloads bucket's S3 API, internal address, e.g. `http://garage-:3900` | + | `DOWNLOADS_BUCKET` | its name, e.g. `downloads.example.com` | + | `DORKNET_SITE_URL` | optional: the DorkNet server's website (e.g. `https://dorknet.example.com`); the page links its Quest installer | 3. **Deploy.** The first image build downloads the Android NDK (~1.5 GB) and takes a while. The node shows as connected in Jenkins once the agent is up. @@ -159,6 +176,7 @@ Add these under **Manage Jenkins → Credentials**, with exactly these IDs: | `quest-keystore-password` | Secret text | its store/key password | | `clientmod-config` | Secret file | your `dorknet-clientmod.json` | | `quest-config` | Secret file | your `dorknet-quest.cfg` | +| `downloads-s3` | Username with password | S3 key for the downloads bucket: access key ID / secret. Give it read and write on that bucket only. | **Use the keystore your players' installs were signed with**, the `dorknet-debug.keystore` from your earlier local builds (password `dorknet`). A @@ -184,6 +202,50 @@ In the repository's **Settings → Webhooks**: - Content type: `application/json` - Events: just the push event +### 8. The downloads page + +The page (`ci/downloads/index.html`) is a static file in an S3 bucket. It +renders the bucket's `manifest.json`, which `publish.py upload` rewrites after +each build. With Garage (the S3 store on the Coolify server): + +1. **Bucket and key.** Create the bucket, serve it as a website, and give + Jenkins a key for that bucket only: + + ```sh + garage bucket create downloads.example.com + garage bucket website --allow downloads.example.com + garage key create jenkins-downloads + garage bucket allow --read --write --key downloads.example.com + ``` + + Store the key as the `downloads-s3` credential. Garage's web endpoint (port + `3902`) serves a bucket whose name is the request's host name, read-only. +2. **Route.** Point the hostname at the web endpoint with a Coolify dynamic + proxy configuration (**Servers → Proxy → Dynamic Configurations**): + + ```yaml + http: + routers: + downloads: + entryPoints: [http] + service: downloads + rule: Host(`downloads.example.com`) + services: + downloads: + loadBalancer: + passHostHeader: true + servers: + - url: "http://garage-:3902" + ``` + +3. **Access.** Put the hostname behind a Cloudflare Access application + **before** the first publish, with the policies of the people who may + download. That group needs no other application. + +Files are uploaded with `Cache-Control: no-cache`, so Cloudflare never serves +an old build after a new one replaces it. Each release keeps one copy of each +file; a new build overwrites it. + ### Disk The agent's `dorknet_cache` volume holds the restored stock builds, several GB diff --git a/ci/jenkins/docker-compose.agent.yml b/ci/jenkins/docker-compose.agent.yml index 1c9045c..e798ce5 100644 --- a/ci/jenkins/docker-compose.agent.yml +++ b/ci/jenkins/docker-compose.agent.yml @@ -28,6 +28,11 @@ services: ARCHIVE_PC_REPO: ${ARCHIVE_PC_REPO:-} ARCHIVE_QUEST_REPO: ${ARCHIVE_QUEST_REPO:-} ARCHIVE_FTP_TLS: ${ARCHIVE_FTP_TLS:-explicit} + # The downloads page's bucket (S3 API, internal address) and the DorkNet + # website the page links to. See ci/downloads/publish.py. + DOWNLOADS_S3_ENDPOINT: ${DOWNLOADS_S3_ENDPOINT:-} + DOWNLOADS_BUCKET: ${DOWNLOADS_BUCKET:-} + DORKNET_SITE_URL: ${DORKNET_SITE_URL:-} volumes: - dorknet_cache:/home/jenkins/dorknet-cache - agent_work:/home/jenkins/agent From ec88a397504e9052e28f2d63deac8c5d7ed7fdc9 Mon Sep 17 00:00:00 2001 From: Alexa <53471801+Alexa-RR@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:55:28 +0200 Subject: [PATCH 2/2] ci: leave the downloads page to the DorkNet site The server now offers the published builds on its own site (Downloads page, Latest build on the Quest page), reading the bucket on the internal network. Drop the standalone index.html and DORKNET_SITE_URL; the bucket needs no public route or access app of its own. --- Jenkinsfile | 6 +- ci/downloads/index.html | 243 ---------------------------- ci/downloads/publish.py | 35 ++-- ci/jenkins/README.md | 76 ++++----- ci/jenkins/docker-compose.agent.yml | 5 +- 5 files changed, 48 insertions(+), 317 deletions(-) delete mode 100644 ci/downloads/index.html diff --git a/Jenkinsfile b/Jenkinsfile index b18f85b..ef53ee8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,7 @@ // On a push, only the platforms whose code changed are rebuilt. A manual run // picks the platforms with PLATFORMS. See ci/jenkins/README.md. // -// Finished builds are renamed for players and published to the downloads page +// Finished builds are renamed for players and published to the downloads bucket // (ci/downloads) unless PUBLISH_DOWNLOADS is off. The build scripts keep their // own file names, which the launcher depends on. @@ -22,7 +22,7 @@ def PC_VERSION_KEY = 'march_2023_03_21' def PC_DEFAULT_TAGS = 'manifest=7490748483298966814' // Steam 2023-03-21 (appVersion 20230317) def QUEST_DEFAULT_TAGS = 'version-code=2128210' // Quest 20230317 -// How this branch's builds are listed on the downloads page. +// How this branch's builds are listed on the site's Downloads page. def RELEASE_ID = '2023.03' def RELEASE_TITLE = 'March 2023' def RELEASE_GAME_VERSION = '2023-03-21' @@ -59,7 +59,7 @@ pipeline { booleanParam(name: 'LIST_SNAPSHOTS', defaultValue: false, description: 'Only list the snapshots matching the tags in both repositories (to find a build), then stop.') booleanParam(name: 'PUBLISH_DOWNLOADS', defaultValue: true, - description: 'Publish what this run built to the downloads page. Push builds publish too.') + description: 'Publish what this run built to the downloads bucket (listed on the site Downloads page). Push builds publish too.') } stages { diff --git a/ci/downloads/index.html b/ci/downloads/index.html deleted file mode 100644 index e5bec93..0000000 --- a/ci/downloads/index.html +++ /dev/null @@ -1,243 +0,0 @@ - - - - - - -DorkNet Downloads - - - - - - - - -
-
- -
DorkNet Downloads
-
- Sign out -
-
- -
-
-

Get Rec Room for DorkNet

-

Pick your platform. Every file is built from the DorkNet source by our build server; the - SHA-256 next to each one lets you check your download.

-
-

Loading builds…

-
-

These builds contain Rec Room's own game files. They're for the DorkNet community only: please don't - re-share them.

-
-
- - - - diff --git a/ci/downloads/publish.py b/ci/downloads/publish.py index 6812ebd..7fe3d70 100644 --- a/ci/downloads/publish.py +++ b/ci/downloads/publish.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 -"""Publish DorkNet client builds to the downloads page. +"""Publish DorkNet client builds to the downloads bucket. stage the build output in out/ -> out/downloads// under the names players see, plus release.json describing those files upload that folder -> the downloads bucket, then merge release.json into - the bucket's manifest.json and upload the page (index.html) + the bucket's manifest.json + +The DorkNet server reads the bucket and offers the files on its site (the +Downloads page, and Latest build on the Quest page); see docs/deploy.md. The build scripts keep their own file names on purpose: the launcher fetches dorknet-clientpatch-.zip from GitHub Releases by that exact name. @@ -15,16 +18,14 @@ Only the published copies are renamed: DorkNet-Quest-/DorkNet-.xapk DorkNet--Quest.xapk DorkNet-Quest-/install.*, README DorkNet--Quest-Installer.zip -The bucket is served read-only as a static website (Garage's web endpoint). -upload talks to its S3 API with rclone; the endpoint, bucket and key come -from the environment: +The server reads the bucket over plain HTTP on the internal network (e.g. +Garage's web endpoint); nothing needs to be public. upload talks to its S3 +API with rclone; the endpoint, bucket and key come from the environment: DOWNLOADS_S3_ENDPOINT S3 API URL, e.g. http://garage-:3900 - DOWNLOADS_BUCKET bucket name, e.g. downloads.example.com + DOWNLOADS_BUCKET bucket name, e.g. dorknet-downloads DOWNLOADS_S3_REGION optional, default garage DOWNLOADS_S3_KEY_ID / DOWNLOADS_S3_SECRET - DORKNET_SITE_URL optional: the DorkNet server's website, linked - from the page for the browser Quest installer python3 ci/downloads/publish.py stage --out out --release 2023.03 \\ --title "March 2023" --game-version 2023-03-21 --version-key march_2023_03_21 @@ -44,12 +45,6 @@ import zipfile from pathlib import Path from typing import NoReturn -HERE = Path(__file__).resolve().parent -REPO = HERE.parent.parent -PAGE_FILES = { # bucket path -> source - "index.html": HERE / "index.html", - "dorknet.svg": REPO / "DorkNet.Server" / "site" / "public" / "brand" / "dorknet.svg", -} NO_CACHE = "Cache-Control: no-cache" INSTALLER_NOTE = """Unzip these files into the same folder as {xapk}, then @@ -223,9 +218,6 @@ def merge(manifest: dict, release: dict) -> dict: releases.sort(key=lambda r: r["id"], reverse=True) out = {**manifest, "releases": releases, "updated": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()} - site = os.environ.get("DORKNET_SITE_URL", "").rstrip("/") - if site: - out["site_url"] = site return out @@ -247,10 +239,7 @@ def cmd_upload(a: argparse.Namespace) -> None: manifest = json.loads(r.stdout) if r.returncode == 0 and r.stdout.strip() else {} manifest = merge(manifest, release) - # 3. the page, then the manifest - for path, src in PAGE_FILES.items(): - step(f"uploading {path}") - rclone(env, "copyto", str(src), f"{bucket}/{path}", "--header-upload", NO_CACHE) + # 3. the manifest, last: it is what makes the new files visible step("uploading manifest.json") rclone(env, "rcat", f"{bucket}/manifest.json", "--header-upload", NO_CACHE, "--header-upload", "Content-Type: application/json", stdin=json.dumps(manifest, indent=2) + "\n") @@ -275,14 +264,14 @@ def main() -> None: s.add_argument("--out", default="out", help="the build output folder (default: out)") s.add_argument("--dest", help="where to stage (default: /downloads)") s.add_argument("--release", required=True, help="release id used in file names and paths, e.g. 2023.03") - s.add_argument("--title", required=True, help='shown on the page, e.g. "March 2023"') + s.add_argument("--title", required=True, help='shown on the Downloads page, e.g. "March 2023"') s.add_argument("--game-version", required=True, help="the stock build's date, e.g. 2023-03-21") s.add_argument("--version-key", required=True, help="the PC build's version key, e.g. march_2023_03_21") s.add_argument("--build", default=os.environ.get("BUILD_NUMBER", ""), help="CI build number (default: $BUILD_NUMBER)") s.add_argument("--commit", default=os.environ.get("GIT_COMMIT", ""), help="source commit (default: $GIT_COMMIT)") s.set_defaults(fn=cmd_stage) - u = sub.add_parser("upload", help="staged folder -> downloads bucket + manifest.json + page") + u = sub.add_parser("upload", help="staged folder -> downloads bucket + manifest.json") u.add_argument("--staged", default="out/downloads", help="the folder `stage` wrote (default: out/downloads)") u.set_defaults(fn=cmd_upload) diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index b6ac455..0540fec 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -11,9 +11,11 @@ The builds run on a **build agent**, not on Jenkins itself: job runs on it through the `dorknet-builder` label. - **Jenkins itself:** stays on the stock image and runs no builds. -Each build is published to the **downloads page** under a name players can +Each build is published to the **downloads bucket** under a name players can read, and the same files are kept as Jenkins artifacts -(`out/downloads/2023.03/`): +(`out/downloads/2023.03/`). The DorkNet server offers them on its site: the +**Downloads** page, and **Latest build** on the **Quest** page +(`docs/deploy.md` → Client downloads). | Download | What it is | Built from | |---|---|---| @@ -26,9 +28,9 @@ The build scripts keep their own names: the launcher downloads `dorknet-clientpatch-.zip` from GitHub Releases by that exact name. `ci/downloads/publish.py` renames only the published copies. -The full PC client and the Quest packages contain game files. The downloads -page is behind Cloudflare Access for that reason; don't publish them -anywhere public. +The full PC client and the Quest packages contain game files. Keep the bucket +internal and the site behind an access proxy; don't publish them anywhere +public. ## Where the stock builds come from @@ -65,10 +67,10 @@ Resolve a table for another build locally with `build.py build --dump-cs`. - **Build with Parameters** runs a manual build: - `PLATFORMS` (both / pc / quest), and `PC_FULL_CLIENT` for the full PC client; - snapshot tags, and per-run repository overrides; - - `PUBLISH_DOWNLOADS` (on by default) to publish to the downloads page. + - `PUBLISH_DOWNLOADS` (on by default) to publish to the downloads bucket. -Push builds publish too, but only the platform they rebuilt: the page keeps -the other platform's last files. +Push builds publish too, but only the platform they rebuilt: the manifest +keeps the other platform's last files. ## Setup @@ -139,8 +141,7 @@ Save, open the node, and copy its **secret**. | `ARCHIVE_QUEST_REPO` | Quest repository folder, e.g. `Rec Room Versions/quest-v2` | | `ARCHIVE_FTP_TLS` | `explicit` (default), `implicit` or `none` | | `DOWNLOADS_S3_ENDPOINT` | the downloads bucket's S3 API, internal address, e.g. `http://garage-:3900` | - | `DOWNLOADS_BUCKET` | its name, e.g. `downloads.example.com` | - | `DORKNET_SITE_URL` | optional: the DorkNet server's website (e.g. `https://dorknet.example.com`); the page links its Quest installer | + | `DOWNLOADS_BUCKET` | its name, e.g. `dorknet-downloads` | 3. **Deploy.** The first image build downloads the Android NDK (~1.5 GB) and takes a while. The node shows as connected in Jenkins once the agent is up. @@ -202,49 +203,34 @@ In the repository's **Settings → Webhooks**: - Content type: `application/json` - Events: just the push event -### 8. The downloads page +### 8. The downloads bucket -The page (`ci/downloads/index.html`) is a static file in an S3 bucket. It -renders the bucket's `manifest.json`, which `publish.py upload` rewrites after -each build. With Garage (the S3 store on the Coolify server): +`publish.py upload` puts the files and a `manifest.json` describing them in +an S3 bucket. The DorkNet server reads that bucket over HTTP on the internal +network and offers the files on its site (`docs/deploy.md` → Client +downloads), so the bucket needs no public route. With Garage (the S3 store on +the Coolify server): -1. **Bucket and key.** Create the bucket, serve it as a website, and give - Jenkins a key for that bucket only: +1. **Bucket and key.** Create the bucket, turn on Garage's website access for + it (the server reads it through the web endpoint, port `3902`, which picks + the bucket by the request's Host header), and give Jenkins a key for that + bucket only: ```sh - garage bucket create downloads.example.com - garage bucket website --allow downloads.example.com + garage bucket create dorknet-downloads + garage bucket website --allow dorknet-downloads garage key create jenkins-downloads - garage bucket allow --read --write --key downloads.example.com + garage bucket allow --read --write --key dorknet-downloads ``` - Store the key as the `downloads-s3` credential. Garage's web endpoint (port - `3902`) serves a bucket whose name is the request's host name, read-only. -2. **Route.** Point the hostname at the web endpoint with a Coolify dynamic - proxy configuration (**Servers → Proxy → Dynamic Configurations**): + Store the key as the `downloads-s3` credential. +2. **Server.** Point the DorkNet server at the web endpoint: + `DorkNet__Downloads__BaseUrl=http://garage-:3902` and + `DorkNet__Downloads__Host=dorknet-downloads`. - ```yaml - http: - routers: - downloads: - entryPoints: [http] - service: downloads - rule: Host(`downloads.example.com`) - services: - downloads: - loadBalancer: - passHostHeader: true - servers: - - url: "http://garage-:3902" - ``` - -3. **Access.** Put the hostname behind a Cloudflare Access application - **before** the first publish, with the policies of the people who may - download. That group needs no other application. - -Files are uploaded with `Cache-Control: no-cache`, so Cloudflare never serves -an old build after a new one replaces it. Each release keeps one copy of each -file; a new build overwrites it. +Each release keeps one copy of each file; a new build overwrites it. The +manifest is uploaded last, so the site only ever lists files that are fully +uploaded. ### Disk diff --git a/ci/jenkins/docker-compose.agent.yml b/ci/jenkins/docker-compose.agent.yml index e798ce5..86efec3 100644 --- a/ci/jenkins/docker-compose.agent.yml +++ b/ci/jenkins/docker-compose.agent.yml @@ -28,11 +28,10 @@ services: ARCHIVE_PC_REPO: ${ARCHIVE_PC_REPO:-} ARCHIVE_QUEST_REPO: ${ARCHIVE_QUEST_REPO:-} ARCHIVE_FTP_TLS: ${ARCHIVE_FTP_TLS:-explicit} - # The downloads page's bucket (S3 API, internal address) and the DorkNet - # website the page links to. See ci/downloads/publish.py. + # The downloads bucket (S3 API, internal address) the builds are + # published to. See ci/downloads/publish.py. DOWNLOADS_S3_ENDPOINT: ${DOWNLOADS_S3_ENDPOINT:-} DOWNLOADS_BUCKET: ${DOWNLOADS_BUCKET:-} - DORKNET_SITE_URL: ${DORKNET_SITE_URL:-} volumes: - dorknet_cache:/home/jenkins/dorknet-cache - agent_work:/home/jenkins/agent