diff --git a/Jenkinsfile b/Jenkinsfile index 6182520..b18f85b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,6 +11,10 @@ // // On a push, only the platforms whose code changed are rebuilt. A manual run // picks the platforms with PLATFORMS. See ci/jenkins/README.md. +// +// Finished builds are renamed for players and published to the downloads page +// (ci/downloads) unless PUBLISH_DOWNLOADS is off. The build scripts keep their +// own file names, which the launcher depends on. // The builds this branch targets. Any other archived build can be picked per // run with the *_SNAPSHOT_TAGS parameters. @@ -18,6 +22,11 @@ def PC_VERSION_KEY = 'march_2023_03_21' def PC_DEFAULT_TAGS = 'manifest=7490748483298966814' // Steam 2023-03-21 (appVersion 20230317) def QUEST_DEFAULT_TAGS = 'version-code=2128210' // Quest 20230317 +// How this branch's builds are listed on the downloads page. +def RELEASE_ID = '2023.03' +def RELEASE_TITLE = 'March 2023' +def RELEASE_GAME_VERSION = '2023-03-21' + def buildPc = false def buildQuest = false @@ -49,6 +58,8 @@ pipeline { description: 'Quest restic repository folder on the FTP. Empty: ARCHIVE_QUEST_REPO set on the agent.') booleanParam(name: 'LIST_SNAPSHOTS', defaultValue: false, description: 'Only list the snapshots matching the tags in both repositories (to find a build), then stop.') + booleanParam(name: 'PUBLISH_DOWNLOADS', defaultValue: true, + description: 'Publish what this run built to the downloads page. Push builds publish too.') } stages { @@ -178,11 +189,35 @@ pipeline { } } } + + stage('Downloads') { + when { expression { buildPc || buildQuest } } + steps { + sh """ + python3 ci/downloads/publish.py stage --out out --release ${RELEASE_ID} \\ + --title '${RELEASE_TITLE}' --game-version ${RELEASE_GAME_VERSION} --version-key ${PC_VERSION_KEY} + """ + script { + if (params.PUBLISH_DOWNLOADS) { + // The bucket's S3 endpoint and name come from the agent's + // environment (DOWNLOADS_S3_ENDPOINT / DOWNLOADS_BUCKET). + withCredentials([usernamePassword(credentialsId: 'downloads-s3', + usernameVariable: 'DOWNLOADS_S3_KEY_ID', passwordVariable: 'DOWNLOADS_S3_SECRET')]) { + sh 'python3 ci/downloads/publish.py upload --staged out/downloads' + } + } else { + echo 'PUBLISH_DOWNLOADS is off: the files are only archived.' + } + } + } + } } post { success { - archiveArtifacts artifacts: 'out/**', allowEmptyArchive: true, fingerprint: true + // The renamed, player-facing files (and release.json): the same set the + // downloads page offers. + archiveArtifacts artifacts: 'out/downloads/**', allowEmptyArchive: true, fingerprint: true } cleanup { // Runs after archiving. Keeps the restic cache and restored stock diff --git a/ci/downloads/index.html b/ci/downloads/index.html new file mode 100644 index 0000000..e5bec93 --- /dev/null +++ b/ci/downloads/index.html @@ -0,0 +1,243 @@ + + + + + + +DorkNet Downloads + + + + + + + + +
+
+ +
DorkNet Downloads
+
+ Sign out +
+
+ +
+
+

Get Rec Room for DorkNet

+

Pick your platform. Every file is built from the DorkNet source by our build server; the + SHA-256 next to each one lets you check your download.

+
+

Loading builds…

+
+

These builds contain Rec Room's own game files. They're for the DorkNet community only: please don't + re-share them.

+
+
+ + + + diff --git a/ci/downloads/publish.py b/ci/downloads/publish.py new file mode 100644 index 0000000..6812ebd --- /dev/null +++ b/ci/downloads/publish.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Publish DorkNet client builds to the downloads page. + + stage the build output in out/ -> out/downloads// under the + names players see, plus release.json describing those files + upload that folder -> the downloads bucket, then merge release.json into + the bucket's manifest.json and upload the page (index.html) + +The build scripts keep their own file names on purpose: the launcher fetches +dorknet-clientpatch-.zip from GitHub Releases by that exact name. +Only the published copies are renamed: + + dorknet-client-.zip DorkNet--PC-Full.zip + dorknet-clientpatch-.zip DorkNet--PC-Patch.zip + DorkNet-Quest-/DorkNet-.xapk DorkNet--Quest.xapk + DorkNet-Quest-/install.*, README DorkNet--Quest-Installer.zip + +The bucket is served read-only as a static website (Garage's web endpoint). +upload talks to its S3 API with rclone; the endpoint, bucket and key come +from the environment: + + DOWNLOADS_S3_ENDPOINT S3 API URL, e.g. http://garage-:3900 + DOWNLOADS_BUCKET bucket name, e.g. downloads.example.com + DOWNLOADS_S3_REGION optional, default garage + DOWNLOADS_S3_KEY_ID / DOWNLOADS_S3_SECRET + DORKNET_SITE_URL optional: the DorkNet server's website, linked + from the page for the browser Quest installer + + python3 ci/downloads/publish.py stage --out out --release 2023.03 \\ + --title "March 2023" --game-version 2023-03-21 --version-key march_2023_03_21 + python3 ci/downloads/publish.py upload --staged out/downloads +""" +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import os +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path +from typing import NoReturn + +HERE = Path(__file__).resolve().parent +REPO = HERE.parent.parent +PAGE_FILES = { # bucket path -> source + "index.html": HERE / "index.html", + "dorknet.svg": REPO / "DorkNet.Server" / "site" / "public" / "brand" / "dorknet.svg", +} +NO_CACHE = "Cache-Control: no-cache" + +INSTALLER_NOTE = """Unzip these files into the same folder as {xapk}, then +run install.cmd (Windows) or ./install.sh (macOS / Linux). + +""" + + +def fail(msg: str) -> NoReturn: + print(f"error: {msg}", file=sys.stderr) + sys.exit(1) + + +def step(msg: str) -> None: + print(f"==> {msg}", flush=True) + + +def human(n: int) -> str: + return f"{n / 2**20:,.1f} MB" if n >= 2**20 else f"{max(1, round(n / 2**10))} KB" + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + while chunk := f.read(1 << 20): + h.update(chunk) + return h.hexdigest() + + +def link_or_copy(src: Path, dst: Path) -> None: + """Hard-link where possible: the full client is several GB.""" + dst.unlink(missing_ok=True) + try: + os.link(src, dst) + except OSError: + shutil.copy2(src, dst) + + +def game_exe(full_zip: Path) -> str | None: + """The game's exe: the one with a matching _Data folder at the root.""" + with zipfile.ZipFile(full_zip) as z: + names = z.namelist() + tops = {n.split("/", 1)[0] for n in names} + exes = sorted(t for t in tops if t.lower().endswith(".exe") and f"{t[:-4]}_Data" in tops) + return exes[0] if exes else None + + +# ---- stage ---------------------------------------------------------------- + +def cmd_stage(a: argparse.Namespace) -> None: + out = Path(a.out) + dest = Path(a.dest) if a.dest else out / "downloads" + rel_dir = dest / a.release + if rel_dir.exists(): + shutil.rmtree(rel_dir) + rel_dir.mkdir(parents=True) + prefix = f"DorkNet-{a.release}" + + staged: list[tuple[str, str, Path]] = [] # (id, platform, staged file) + extra: dict[str, str] = {} + + full = out / "pc" / f"dorknet-client-{a.version_key}.zip" + patch = out / "pc" / f"dorknet-clientpatch-{a.version_key}.zip" + if full.is_file(): + dst = rel_dir / f"{prefix}-PC-Full.zip" + link_or_copy(full, dst) + staged.append(("pc-full", "pc", dst)) + exe = game_exe(full) + if exe: + extra["pc_exe"] = exe + if patch.is_file(): + dst = rel_dir / f"{prefix}-PC-Patch.zip" + link_or_copy(patch, dst) + staged.append(("pc-patch", "pc", dst)) + + packages = sorted((out / "quest").glob("DorkNet-Quest-*/")) + if len(packages) > 1: + fail(f"more than one Quest package folder in {out / 'quest'}: {', '.join(p.name for p in packages)}") + if packages: + pkg = packages[0] + xapks = sorted(pkg.glob("*.xapk")) + if len(xapks) != 1: + fail(f"expected one .xapk in {pkg}, found {len(xapks)}") + xapk = rel_dir / f"{prefix}-Quest.xapk" + link_or_copy(xapks[0], xapk) + staged.append(("quest", "quest", xapk)) + + installer = rel_dir / f"{prefix}-Quest-Installer.zip" + with zipfile.ZipFile(installer, "w", zipfile.ZIP_DEFLATED) as z: + for name in ("install.cmd", "install.ps1", "install.sh"): + src = pkg / name + if not src.is_file(): + fail(f"{src} missing") + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.external_attr = (0o755 if name.endswith(".sh") else 0o644) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, src.read_bytes()) + readme = pkg / "README.txt" + text = INSTALLER_NOTE.format(xapk=xapk.name) + (readme.read_text(encoding="utf-8") if readme.is_file() else "") + info = zipfile.ZipInfo("README.txt", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, text) + staged.append(("quest-installer", "quest", installer)) + + if not staged: + fail(f"nothing to publish in {out}") + + built_at = datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat() + files = [] + for fid, platform, path in staged: + step(f"{path.name}: hashing") + files.append({ + "id": fid, + "platform": platform, + "name": path.name, + "path": f"{a.release}/{path.name}", + "size": path.stat().st_size, + "sha256": sha256(path), + "build": a.build, + "commit": a.commit, + "built_at": built_at, + }) + release = {"id": a.release, "title": a.title, "game_version": a.game_version, **extra, "files": files} + (dest / "release.json").write_text(json.dumps(release, indent=2) + "\n", encoding="utf-8") + print() + for f in files: + print(f" {f['name']} {human(f['size'])} {f['sha256'][:16]}…") + print(f"Staged {len(files)} file(s) in {rel_dir}") + + +# ---- upload --------------------------------------------------------------- + +def rclone_env() -> dict[str, str]: + env = dict(os.environ) + for k in ("DOWNLOADS_S3_ENDPOINT", "DOWNLOADS_BUCKET", "DOWNLOADS_S3_KEY_ID", "DOWNLOADS_S3_SECRET"): + if not env.get(k): + fail(f"{k} is not set") + env.update({ + "RCLONE_CONFIG_DL_TYPE": "s3", + "RCLONE_CONFIG_DL_PROVIDER": "Other", + "RCLONE_CONFIG_DL_ENDPOINT": env["DOWNLOADS_S3_ENDPOINT"], + "RCLONE_CONFIG_DL_REGION": env.get("DOWNLOADS_S3_REGION") or "garage", + "RCLONE_CONFIG_DL_ACCESS_KEY_ID": env["DOWNLOADS_S3_KEY_ID"], + "RCLONE_CONFIG_DL_SECRET_ACCESS_KEY": env["DOWNLOADS_S3_SECRET"], + "RCLONE_CONFIG_DL_FORCE_PATH_STYLE": "true", + "RCLONE_CONFIG_DL_NO_CHECK_BUCKET": "true", + }) + return env + + +def rclone(env: dict[str, str], *args: str, stdin: str | None = None, check: bool = True) -> subprocess.CompletedProcess: + r = subprocess.run(["rclone", *args], env=env, input=stdin, capture_output=True, text=True) + if check and r.returncode != 0: + fail(f"rclone {args[0]} failed ({r.returncode}): {r.stderr.strip()}") + return r + + +def merge(manifest: dict, release: dict) -> dict: + """Replace this release's files that were just built; keep the others + (a push build may rebuild only one platform).""" + releases = [r for r in manifest.get("releases", []) if r.get("id") != release["id"]] + old = next((r for r in manifest.get("releases", []) if r.get("id") == release["id"]), {}) + new_ids = {f["id"] for f in release["files"]} + kept = [f for f in old.get("files", []) if f.get("id") not in new_ids] + merged = {**old, **release, "files": kept + release["files"]} + if "pc_exe" not in release and "pc_exe" in old: + merged["pc_exe"] = old["pc_exe"] + order = ["pc-full", "pc-patch", "quest", "quest-installer"] + merged["files"].sort(key=lambda f: order.index(f["id"]) if f["id"] in order else len(order)) + releases.append(merged) + releases.sort(key=lambda r: r["id"], reverse=True) + out = {**manifest, "releases": releases, + "updated": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()} + site = os.environ.get("DORKNET_SITE_URL", "").rstrip("/") + if site: + out["site_url"] = site + return out + + +def cmd_upload(a: argparse.Namespace) -> None: + staged = Path(a.staged) + release = json.loads((staged / "release.json").read_text(encoding="utf-8")) + env = rclone_env() + bucket = f"dl:{env['DOWNLOADS_BUCKET']}" + rel = release["id"] + + # 1. the files, before the manifest that points at them + for f in release["files"]: + step(f"uploading {f['name']} ({human(f['size'])})") + rclone(env, "copyto", str(staged / f["path"]), f"{bucket}/{f['path']}", + "--header-upload", NO_CACHE, "--s3-upload-concurrency", "8", "--s3-chunk-size", "64M") + + # 2. merge into the live manifest + r = rclone(env, "cat", f"{bucket}/manifest.json", check=False) + manifest = json.loads(r.stdout) if r.returncode == 0 and r.stdout.strip() else {} + manifest = merge(manifest, release) + + # 3. the page, then the manifest + for path, src in PAGE_FILES.items(): + step(f"uploading {path}") + rclone(env, "copyto", str(src), f"{bucket}/{path}", "--header-upload", NO_CACHE) + step("uploading manifest.json") + rclone(env, "rcat", f"{bucket}/manifest.json", "--header-upload", NO_CACHE, + "--header-upload", "Content-Type: application/json", stdin=json.dumps(manifest, indent=2) + "\n") + + # 4. drop files of this release the manifest no longer lists (renames) + listed = {f["path"] for r in manifest["releases"] if r["id"] == rel for f in r["files"]} + r = rclone(env, "lsf", f"{bucket}/{rel}/", "--files-only", check=False) + for name in r.stdout.split(): + if f"{rel}/{name}" not in listed: + step(f"removing stale {rel}/{name}") + rclone(env, "deletefile", f"{bucket}/{rel}/{name}") + print(f"Published {len(release['files'])} file(s) for {release['title']}.") + + +# ---- cli ------------------------------------------------------------------ + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = p.add_subparsers(dest="cmd", required=True) + + s = sub.add_parser("stage", help="out/ -> out/downloads// + release.json") + s.add_argument("--out", default="out", help="the build output folder (default: out)") + s.add_argument("--dest", help="where to stage (default: /downloads)") + s.add_argument("--release", required=True, help="release id used in file names and paths, e.g. 2023.03") + s.add_argument("--title", required=True, help='shown on the page, e.g. "March 2023"') + s.add_argument("--game-version", required=True, help="the stock build's date, e.g. 2023-03-21") + s.add_argument("--version-key", required=True, help="the PC build's version key, e.g. march_2023_03_21") + s.add_argument("--build", default=os.environ.get("BUILD_NUMBER", ""), help="CI build number (default: $BUILD_NUMBER)") + s.add_argument("--commit", default=os.environ.get("GIT_COMMIT", ""), help="source commit (default: $GIT_COMMIT)") + s.set_defaults(fn=cmd_stage) + + u = sub.add_parser("upload", help="staged folder -> downloads bucket + manifest.json + page") + u.add_argument("--staged", default="out/downloads", help="the folder `stage` wrote (default: out/downloads)") + u.set_defaults(fn=cmd_upload) + + a = p.parse_args() + a.fn(a) + + +if __name__ == "__main__": + main() diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index 2b6b8e3..b6ac455 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -11,14 +11,24 @@ The builds run on a **build agent**, not on Jenkins itself: job runs on it through the `dorknet-builder` label. - **Jenkins itself:** stays on the stock image and runs no builds. -| Platform | Output (Jenkins artifacts) | How | -|---|---|---| -| PC | `out/pc/dorknet-clientpatch-march_2023_03_21.zip`: MelonLoader, mod, config template, Steam stub, `manifest.json`; no game files | `tools/build_pc.py patcher` | -| PC | `out/pc/dorknet-client-march_2023_03_21.zip`: the stock Steam build with the patcher applied and your config; ready to run | `tools/build_pc.py client` | -| Quest | `out/quest/dorknet-recroom-.apk` and `out/quest/DorkNet-Quest-/` (`.xapk` + installers) | `DorkNet.Quest/scripts/build.py build` | +Each build is published to the **downloads page** under a name players can +read, and the same files are kept as Jenkins artifacts +(`out/downloads/2023.03/`): -The full PC client and the Quest packages contain game files, so keep them on -Jenkins. Don't publish them anywhere public. +| Download | What it is | Built from | +|---|---|---| +| `DorkNet-2023.03-PC-Full.zip` | the stock Steam build with the patcher applied and your config; ready to run | `tools/build_pc.py client` → `dorknet-client-march_2023_03_21.zip` | +| `DorkNet-2023.03-PC-Patch.zip` | MelonLoader, mod, config template, Steam stub, `manifest.json`; no game files | `tools/build_pc.py patcher` → `dorknet-clientpatch-march_2023_03_21.zip` | +| `DorkNet-2023.03-Quest.xapk` | patched APK + OBB, for the browser installer or SideQuest | `DorkNet.Quest/scripts/build.py build` → `DorkNet-Quest-/DorkNet-.xapk` | +| `DorkNet-2023.03-Quest-Installer.zip` | `install.cmd` / `install.ps1` / `install.sh` + README, unzipped next to the `.xapk` | the same package folder | + +The build scripts keep their own names: the launcher downloads +`dorknet-clientpatch-.zip` from GitHub Releases by that exact +name. `ci/downloads/publish.py` renames only the published copies. + +The full PC client and the Quest packages contain game files. The downloads +page is behind Cloudflare Access for that reason; don't publish them +anywhere public. ## Where the stock builds come from @@ -54,7 +64,11 @@ Resolve a table for another build locally with `build.py build --dump-cs`. - `Jenkinsfile` and `ci/jenkins/` rebuild both. - **Build with Parameters** runs a manual build: - `PLATFORMS` (both / pc / quest), and `PC_FULL_CLIENT` for the full PC client; - - snapshot tags, and per-run repository overrides. + - snapshot tags, and per-run repository overrides; + - `PUBLISH_DOWNLOADS` (on by default) to publish to the downloads page. + +Push builds publish too, but only the platform they rebuilt: the page keeps +the other platform's last files. ## Setup @@ -124,6 +138,9 @@ Save, open the node, and copy its **secret**. | `ARCHIVE_PC_REPO` | Steam repository folder on the FTP, e.g. `Rec Room Versions/steam-v2` (plain text, not URL-encoded) | | `ARCHIVE_QUEST_REPO` | Quest repository folder, e.g. `Rec Room Versions/quest-v2` | | `ARCHIVE_FTP_TLS` | `explicit` (default), `implicit` or `none` | + | `DOWNLOADS_S3_ENDPOINT` | the downloads bucket's S3 API, internal address, e.g. `http://garage-:3900` | + | `DOWNLOADS_BUCKET` | its name, e.g. `downloads.example.com` | + | `DORKNET_SITE_URL` | optional: the DorkNet server's website (e.g. `https://dorknet.example.com`); the page links its Quest installer | 3. **Deploy.** The first image build downloads the Android NDK (~1.5 GB) and takes a while. The node shows as connected in Jenkins once the agent is up. @@ -159,6 +176,7 @@ Add these under **Manage Jenkins → Credentials**, with exactly these IDs: | `quest-keystore-password` | Secret text | its store/key password | | `clientmod-config` | Secret file | your `dorknet-clientmod.json` | | `quest-config` | Secret file | your `dorknet-quest.cfg` | +| `downloads-s3` | Username with password | S3 key for the downloads bucket: access key ID / secret. Give it read and write on that bucket only. | **Use the keystore your players' installs were signed with**, the `dorknet-debug.keystore` from your earlier local builds (password `dorknet`). A @@ -184,6 +202,50 @@ In the repository's **Settings → Webhooks**: - Content type: `application/json` - Events: just the push event +### 8. The downloads page + +The page (`ci/downloads/index.html`) is a static file in an S3 bucket. It +renders the bucket's `manifest.json`, which `publish.py upload` rewrites after +each build. With Garage (the S3 store on the Coolify server): + +1. **Bucket and key.** Create the bucket, serve it as a website, and give + Jenkins a key for that bucket only: + + ```sh + garage bucket create downloads.example.com + garage bucket website --allow downloads.example.com + garage key create jenkins-downloads + garage bucket allow --read --write --key downloads.example.com + ``` + + Store the key as the `downloads-s3` credential. Garage's web endpoint (port + `3902`) serves a bucket whose name is the request's host name, read-only. +2. **Route.** Point the hostname at the web endpoint with a Coolify dynamic + proxy configuration (**Servers → Proxy → Dynamic Configurations**): + + ```yaml + http: + routers: + downloads: + entryPoints: [http] + service: downloads + rule: Host(`downloads.example.com`) + services: + downloads: + loadBalancer: + passHostHeader: true + servers: + - url: "http://garage-:3902" + ``` + +3. **Access.** Put the hostname behind a Cloudflare Access application + **before** the first publish, with the policies of the people who may + download. That group needs no other application. + +Files are uploaded with `Cache-Control: no-cache`, so Cloudflare never serves +an old build after a new one replaces it. Each release keeps one copy of each +file; a new build overwrites it. + ### Disk The agent's `dorknet_cache` volume holds the restored stock builds, several GB diff --git a/ci/jenkins/docker-compose.agent.yml b/ci/jenkins/docker-compose.agent.yml index 1c9045c..e798ce5 100644 --- a/ci/jenkins/docker-compose.agent.yml +++ b/ci/jenkins/docker-compose.agent.yml @@ -28,6 +28,11 @@ services: ARCHIVE_PC_REPO: ${ARCHIVE_PC_REPO:-} ARCHIVE_QUEST_REPO: ${ARCHIVE_QUEST_REPO:-} ARCHIVE_FTP_TLS: ${ARCHIVE_FTP_TLS:-explicit} + # The downloads page's bucket (S3 API, internal address) and the DorkNet + # website the page links to. See ci/downloads/publish.py. + DOWNLOADS_S3_ENDPOINT: ${DOWNLOADS_S3_ENDPOINT:-} + DOWNLOADS_BUCKET: ${DOWNLOADS_BUCKET:-} + DORKNET_SITE_URL: ${DORKNET_SITE_URL:-} volumes: - dorknet_cache:/home/jenkins/dorknet-cache - agent_work:/home/jenkins/agent