diff --git a/Jenkinsfile b/Jenkinsfile index b18f85b..ef53ee8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,7 +12,7 @@ // On a push, only the platforms whose code changed are rebuilt. A manual run // picks the platforms with PLATFORMS. See ci/jenkins/README.md. // -// Finished builds are renamed for players and published to the downloads page +// Finished builds are renamed for players and published to the downloads bucket // (ci/downloads) unless PUBLISH_DOWNLOADS is off. The build scripts keep their // own file names, which the launcher depends on. @@ -22,7 +22,7 @@ def PC_VERSION_KEY = 'march_2023_03_21' def PC_DEFAULT_TAGS = 'manifest=7490748483298966814' // Steam 2023-03-21 (appVersion 20230317) def QUEST_DEFAULT_TAGS = 'version-code=2128210' // Quest 20230317 -// How this branch's builds are listed on the downloads page. +// How this branch's builds are listed on the site's Downloads page. def RELEASE_ID = '2023.03' def RELEASE_TITLE = 'March 2023' def RELEASE_GAME_VERSION = '2023-03-21' @@ -59,7 +59,7 @@ pipeline { booleanParam(name: 'LIST_SNAPSHOTS', defaultValue: false, description: 'Only list the snapshots matching the tags in both repositories (to find a build), then stop.') booleanParam(name: 'PUBLISH_DOWNLOADS', defaultValue: true, - description: 'Publish what this run built to the downloads page. Push builds publish too.') + description: 'Publish what this run built to the downloads bucket (listed on the site Downloads page). Push builds publish too.') } stages { diff --git a/ci/downloads/index.html b/ci/downloads/index.html deleted file mode 100644 index e5bec93..0000000 --- a/ci/downloads/index.html +++ /dev/null @@ -1,243 +0,0 @@ - - - - - - -DorkNet Downloads - - - - - - - - -
-
- -
DorkNet Downloads
-
- Sign out -
-
- -
-
-

Get Rec Room for DorkNet

-

Pick your platform. Every file is built from the DorkNet source by our build server; the - SHA-256 next to each one lets you check your download.

-
-

Loading builds…

-
-

These builds contain Rec Room's own game files. They're for the DorkNet community only: please don't - re-share them.

-
-
- - - - diff --git a/ci/downloads/publish.py b/ci/downloads/publish.py index 6812ebd..7fe3d70 100644 --- a/ci/downloads/publish.py +++ b/ci/downloads/publish.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 -"""Publish DorkNet client builds to the downloads page. +"""Publish DorkNet client builds to the downloads bucket. stage the build output in out/ -> out/downloads// under the names players see, plus release.json describing those files upload that folder -> the downloads bucket, then merge release.json into - the bucket's manifest.json and upload the page (index.html) + the bucket's manifest.json + +The DorkNet server reads the bucket and offers the files on its site (the +Downloads page, and Latest build on the Quest page); see docs/deploy.md. The build scripts keep their own file names on purpose: the launcher fetches dorknet-clientpatch-.zip from GitHub Releases by that exact name. @@ -15,16 +18,14 @@ Only the published copies are renamed: DorkNet-Quest-/DorkNet-.xapk DorkNet--Quest.xapk DorkNet-Quest-/install.*, README DorkNet--Quest-Installer.zip -The bucket is served read-only as a static website (Garage's web endpoint). -upload talks to its S3 API with rclone; the endpoint, bucket and key come -from the environment: +The server reads the bucket over plain HTTP on the internal network (e.g. +Garage's web endpoint); nothing needs to be public. upload talks to its S3 +API with rclone; the endpoint, bucket and key come from the environment: DOWNLOADS_S3_ENDPOINT S3 API URL, e.g. http://garage-:3900 - DOWNLOADS_BUCKET bucket name, e.g. downloads.example.com + DOWNLOADS_BUCKET bucket name, e.g. dorknet-downloads DOWNLOADS_S3_REGION optional, default garage DOWNLOADS_S3_KEY_ID / DOWNLOADS_S3_SECRET - DORKNET_SITE_URL optional: the DorkNet server's website, linked - from the page for the browser Quest installer python3 ci/downloads/publish.py stage --out out --release 2023.03 \\ --title "March 2023" --game-version 2023-03-21 --version-key march_2023_03_21 @@ -44,12 +45,6 @@ import zipfile from pathlib import Path from typing import NoReturn -HERE = Path(__file__).resolve().parent -REPO = HERE.parent.parent -PAGE_FILES = { # bucket path -> source - "index.html": HERE / "index.html", - "dorknet.svg": REPO / "DorkNet.Server" / "site" / "public" / "brand" / "dorknet.svg", -} NO_CACHE = "Cache-Control: no-cache" INSTALLER_NOTE = """Unzip these files into the same folder as {xapk}, then @@ -223,9 +218,6 @@ def merge(manifest: dict, release: dict) -> dict: releases.sort(key=lambda r: r["id"], reverse=True) out = {**manifest, "releases": releases, "updated": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()} - site = os.environ.get("DORKNET_SITE_URL", "").rstrip("/") - if site: - out["site_url"] = site return out @@ -247,10 +239,7 @@ def cmd_upload(a: argparse.Namespace) -> None: manifest = json.loads(r.stdout) if r.returncode == 0 and r.stdout.strip() else {} manifest = merge(manifest, release) - # 3. the page, then the manifest - for path, src in PAGE_FILES.items(): - step(f"uploading {path}") - rclone(env, "copyto", str(src), f"{bucket}/{path}", "--header-upload", NO_CACHE) + # 3. the manifest, last: it is what makes the new files visible step("uploading manifest.json") rclone(env, "rcat", f"{bucket}/manifest.json", "--header-upload", NO_CACHE, "--header-upload", "Content-Type: application/json", stdin=json.dumps(manifest, indent=2) + "\n") @@ -275,14 +264,14 @@ def main() -> None: s.add_argument("--out", default="out", help="the build output folder (default: out)") s.add_argument("--dest", help="where to stage (default: /downloads)") s.add_argument("--release", required=True, help="release id used in file names and paths, e.g. 2023.03") - s.add_argument("--title", required=True, help='shown on the page, e.g. "March 2023"') + s.add_argument("--title", required=True, help='shown on the Downloads page, e.g. "March 2023"') s.add_argument("--game-version", required=True, help="the stock build's date, e.g. 2023-03-21") s.add_argument("--version-key", required=True, help="the PC build's version key, e.g. march_2023_03_21") s.add_argument("--build", default=os.environ.get("BUILD_NUMBER", ""), help="CI build number (default: $BUILD_NUMBER)") s.add_argument("--commit", default=os.environ.get("GIT_COMMIT", ""), help="source commit (default: $GIT_COMMIT)") s.set_defaults(fn=cmd_stage) - u = sub.add_parser("upload", help="staged folder -> downloads bucket + manifest.json + page") + u = sub.add_parser("upload", help="staged folder -> downloads bucket + manifest.json") u.add_argument("--staged", default="out/downloads", help="the folder `stage` wrote (default: out/downloads)") u.set_defaults(fn=cmd_upload) diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index b6ac455..0540fec 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -11,9 +11,11 @@ The builds run on a **build agent**, not on Jenkins itself: job runs on it through the `dorknet-builder` label. - **Jenkins itself:** stays on the stock image and runs no builds. -Each build is published to the **downloads page** under a name players can +Each build is published to the **downloads bucket** under a name players can read, and the same files are kept as Jenkins artifacts -(`out/downloads/2023.03/`): +(`out/downloads/2023.03/`). The DorkNet server offers them on its site: the +**Downloads** page, and **Latest build** on the **Quest** page +(`docs/deploy.md` → Client downloads). | Download | What it is | Built from | |---|---|---| @@ -26,9 +28,9 @@ The build scripts keep their own names: the launcher downloads `dorknet-clientpatch-.zip` from GitHub Releases by that exact name. `ci/downloads/publish.py` renames only the published copies. -The full PC client and the Quest packages contain game files. The downloads -page is behind Cloudflare Access for that reason; don't publish them -anywhere public. +The full PC client and the Quest packages contain game files. Keep the bucket +internal and the site behind an access proxy; don't publish them anywhere +public. ## Where the stock builds come from @@ -65,10 +67,10 @@ Resolve a table for another build locally with `build.py build --dump-cs`. - **Build with Parameters** runs a manual build: - `PLATFORMS` (both / pc / quest), and `PC_FULL_CLIENT` for the full PC client; - snapshot tags, and per-run repository overrides; - - `PUBLISH_DOWNLOADS` (on by default) to publish to the downloads page. + - `PUBLISH_DOWNLOADS` (on by default) to publish to the downloads bucket. -Push builds publish too, but only the platform they rebuilt: the page keeps -the other platform's last files. +Push builds publish too, but only the platform they rebuilt: the manifest +keeps the other platform's last files. ## Setup @@ -139,8 +141,7 @@ Save, open the node, and copy its **secret**. | `ARCHIVE_QUEST_REPO` | Quest repository folder, e.g. `Rec Room Versions/quest-v2` | | `ARCHIVE_FTP_TLS` | `explicit` (default), `implicit` or `none` | | `DOWNLOADS_S3_ENDPOINT` | the downloads bucket's S3 API, internal address, e.g. `http://garage-:3900` | - | `DOWNLOADS_BUCKET` | its name, e.g. `downloads.example.com` | - | `DORKNET_SITE_URL` | optional: the DorkNet server's website (e.g. `https://dorknet.example.com`); the page links its Quest installer | + | `DOWNLOADS_BUCKET` | its name, e.g. `dorknet-downloads` | 3. **Deploy.** The first image build downloads the Android NDK (~1.5 GB) and takes a while. The node shows as connected in Jenkins once the agent is up. @@ -202,49 +203,34 @@ In the repository's **Settings → Webhooks**: - Content type: `application/json` - Events: just the push event -### 8. The downloads page +### 8. The downloads bucket -The page (`ci/downloads/index.html`) is a static file in an S3 bucket. It -renders the bucket's `manifest.json`, which `publish.py upload` rewrites after -each build. With Garage (the S3 store on the Coolify server): +`publish.py upload` puts the files and a `manifest.json` describing them in +an S3 bucket. The DorkNet server reads that bucket over HTTP on the internal +network and offers the files on its site (`docs/deploy.md` → Client +downloads), so the bucket needs no public route. With Garage (the S3 store on +the Coolify server): -1. **Bucket and key.** Create the bucket, serve it as a website, and give - Jenkins a key for that bucket only: +1. **Bucket and key.** Create the bucket, turn on Garage's website access for + it (the server reads it through the web endpoint, port `3902`, which picks + the bucket by the request's Host header), and give Jenkins a key for that + bucket only: ```sh - garage bucket create downloads.example.com - garage bucket website --allow downloads.example.com + garage bucket create dorknet-downloads + garage bucket website --allow dorknet-downloads garage key create jenkins-downloads - garage bucket allow --read --write --key downloads.example.com + garage bucket allow --read --write --key dorknet-downloads ``` - Store the key as the `downloads-s3` credential. Garage's web endpoint (port - `3902`) serves a bucket whose name is the request's host name, read-only. -2. **Route.** Point the hostname at the web endpoint with a Coolify dynamic - proxy configuration (**Servers → Proxy → Dynamic Configurations**): + Store the key as the `downloads-s3` credential. +2. **Server.** Point the DorkNet server at the web endpoint: + `DorkNet__Downloads__BaseUrl=http://garage-:3902` and + `DorkNet__Downloads__Host=dorknet-downloads`. - ```yaml - http: - routers: - downloads: - entryPoints: [http] - service: downloads - rule: Host(`downloads.example.com`) - services: - downloads: - loadBalancer: - passHostHeader: true - servers: - - url: "http://garage-:3902" - ``` - -3. **Access.** Put the hostname behind a Cloudflare Access application - **before** the first publish, with the policies of the people who may - download. That group needs no other application. - -Files are uploaded with `Cache-Control: no-cache`, so Cloudflare never serves -an old build after a new one replaces it. Each release keeps one copy of each -file; a new build overwrites it. +Each release keeps one copy of each file; a new build overwrites it. The +manifest is uploaded last, so the site only ever lists files that are fully +uploaded. ### Disk diff --git a/ci/jenkins/docker-compose.agent.yml b/ci/jenkins/docker-compose.agent.yml index e798ce5..86efec3 100644 --- a/ci/jenkins/docker-compose.agent.yml +++ b/ci/jenkins/docker-compose.agent.yml @@ -28,11 +28,10 @@ services: ARCHIVE_PC_REPO: ${ARCHIVE_PC_REPO:-} ARCHIVE_QUEST_REPO: ${ARCHIVE_QUEST_REPO:-} ARCHIVE_FTP_TLS: ${ARCHIVE_FTP_TLS:-explicit} - # The downloads page's bucket (S3 API, internal address) and the DorkNet - # website the page links to. See ci/downloads/publish.py. + # The downloads bucket (S3 API, internal address) the builds are + # published to. See ci/downloads/publish.py. DOWNLOADS_S3_ENDPOINT: ${DOWNLOADS_S3_ENDPOINT:-} DOWNLOADS_BUCKET: ${DOWNLOADS_BUCKET:-} - DORKNET_SITE_URL: ${DORKNET_SITE_URL:-} volumes: - dorknet_cache:/home/jenkins/dorknet-cache - agent_work:/home/jenkins/agent