diff --git a/Jenkinsfile b/Jenkinsfile index 9609d05..6182520 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,14 +1,13 @@ // DorkNet client builds — PC (patcher zip + full client) and Quest (.xapk). // -// Runs on the Jenkins image from ci/jenkins (the stock image plus Python with -// LIEF, .NET, the Android NDK/build-tools, apktool, restic and rclone). +// Runs on the "dorknet-builder" agent from ci/jenkins (an inbound agent with +// Python + LIEF, .NET, the Android NDK/build-tools, apktool, restic and rclone). // // Stock builds come out of the restic build archive on the FTP. Where it lives // is server configuration, not part of the repo: the FTP host, login and the // restic password are Jenkins credentials, and the repository folders come -// from the Jenkins container's environment (ARCHIVE_PC_REPO / -// ARCHIVE_QUEST_REPO), each overridable per run. A build is picked by -// snapshot tags. +// from the agent's environment (ARCHIVE_PC_REPO / ARCHIVE_QUEST_REPO), each +// overridable per run. A build is picked by snapshot tags. // // On a push, only the platforms whose code changed are rebuilt. A manual run // picks the platforms with PLATFORMS. See ci/jenkins/README.md. @@ -23,7 +22,7 @@ def buildPc = false def buildQuest = false pipeline { - agent any + agent { label 'dorknet-builder' } options { timestamps() @@ -45,9 +44,9 @@ pipeline { string(name: 'QUEST_SNAPSHOT_TAGS', defaultValue: '', description: "restic tags picking the stock Quest build, comma = AND. Empty: ${QUEST_DEFAULT_TAGS}") string(name: 'PC_REPO', defaultValue: '', - description: 'PC restic repository folder on the FTP. Empty: ARCHIVE_PC_REPO from the Jenkins environment.') + description: 'PC restic repository folder on the FTP. Empty: ARCHIVE_PC_REPO set on the agent.') string(name: 'QUEST_REPO', defaultValue: '', - description: 'Quest restic repository folder on the FTP. Empty: ARCHIVE_QUEST_REPO from the Jenkins environment.') + description: 'Quest restic repository folder on the FTP. Empty: ARCHIVE_QUEST_REPO set on the agent.') booleanParam(name: 'LIST_SNAPSHOTS', defaultValue: false, description: 'Only list the snapshots matching the tags in both repositories (to find a build), then stop.') } @@ -88,8 +87,8 @@ pipeline { "PC_TAGS=${params.PC_SNAPSHOT_TAGS}", "QUEST_TAGS=${params.QUEST_SNAPSHOT_TAGS}"]) { sh ''' export RESTIC_CACHE_DIR="$DORKNET_CACHE/restic" - pc="${PC_REPO_OVERRIDE:-${ARCHIVE_PC_REPO:?set ARCHIVE_PC_REPO on Jenkins or PC_REPO on the run}}" - quest="${QUEST_REPO_OVERRIDE:-${ARCHIVE_QUEST_REPO:?set ARCHIVE_QUEST_REPO on Jenkins or QUEST_REPO on the run}}" + pc="${PC_REPO_OVERRIDE:-${ARCHIVE_PC_REPO:?set ARCHIVE_PC_REPO on the agent or PC_REPO on the run}}" + quest="${QUEST_REPO_OVERRIDE:-${ARCHIVE_QUEST_REPO:?set ARCHIVE_QUEST_REPO on the agent or QUEST_REPO on the run}}" echo "== PC: $pc"; archive-restore list "$pc" "$PC_TAGS" echo "== Quest: $quest"; archive-restore list "$quest" "$QUEST_TAGS" ''' @@ -113,7 +112,7 @@ pipeline { withEnv(["REPO_OVERRIDE=${params.PC_REPO}", "TAGS=${params.PC_SNAPSHOT_TAGS ?: PC_DEFAULT_TAGS}"]) { sh(returnStdout: true, script: ''' export RESTIC_CACHE_DIR="$DORKNET_CACHE/restic" - repo="${REPO_OVERRIDE:-${ARCHIVE_PC_REPO:?set ARCHIVE_PC_REPO on Jenkins or PC_REPO on the run}}" + repo="${REPO_OVERRIDE:-${ARCHIVE_PC_REPO:?set ARCHIVE_PC_REPO on the agent or PC_REPO on the run}}" archive-restore restore "$repo" "$TAGS" "$DORKNET_CACHE/pc" --exclude /.DepotDownloader ''').trim().readLines().last() } @@ -141,7 +140,7 @@ pipeline { withEnv(["REPO_OVERRIDE=${params.QUEST_REPO}", "TAGS=${params.QUEST_SNAPSHOT_TAGS ?: QUEST_DEFAULT_TAGS}"]) { sh(returnStdout: true, script: ''' export RESTIC_CACHE_DIR="$DORKNET_CACHE/restic" - repo="${REPO_OVERRIDE:-${ARCHIVE_QUEST_REPO:?set ARCHIVE_QUEST_REPO on Jenkins or QUEST_REPO on the run}}" + repo="${REPO_OVERRIDE:-${ARCHIVE_QUEST_REPO:?set ARCHIVE_QUEST_REPO on the agent or QUEST_REPO on the run}}" archive-restore restore "$repo" "$TAGS" "$DORKNET_CACHE/quest" ''').trim().readLines().last() } @@ -187,7 +186,7 @@ pipeline { } cleanup { // Runs after archiving. Keeps the restic cache and restored stock - // builds ($DORKNET_CACHE in jenkins_home, outside the workspace); drops + // builds ($DORKNET_CACHE on the agent, outside the workspace); drops // the workspace's build output and scratch. cleanWs(deleteDirs: true, notFailBuild: true) } diff --git a/ci/jenkins/Dockerfile b/ci/jenkins/Dockerfile index 78486f8..85c9b6b 100644 --- a/ci/jenkins/Dockerfile +++ b/ci/jenkins/Dockerfile @@ -1,7 +1,7 @@ -# DorkNet Jenkins: the stock Jenkins image plus everything the PC and Quest -# client builds need, so the job (the Jenkinsfile at the repo root) runs right -# on the controller. Jobs, plugins and settings stay in the jenkins_home -# volume; only the image changes. +# DorkNet build agent: a Jenkins inbound agent with everything the PC and +# Quest client builds need. It runs next to Jenkins on Coolify, connects to +# the controller over WebSocket, and runs the jobs labelled "dorknet-builder" +# (the Jenkinsfile at the repo root). Jenkins itself stays on the stock image. # # PC Python + .NET SDK tools/build_pc.py # Quest Python/LIEF + JDK + Android NDK, build-tools, apktool @@ -9,13 +9,13 @@ # Builds restic + rclone stock builds restored from the restic # archive on the FTP (archive-restore.sh) # +# Java 25: Jenkins supports it until Sep 2029 (Java 21 only until Sep 2027). +# Keep the agent's Java the same major version as the controller's. +# # See ci/jenkins/README.md. -# Match this to the Jenkins image your Dokploy service runs today, so the -# upgrade keeps the same Jenkins version and Java: e.g. -# --build-arg JENKINS_IMAGE=jenkins/jenkins:2.492.3-lts-jdk17 -ARG JENKINS_IMAGE=jenkins/jenkins:lts-jdk17 -FROM ${JENKINS_IMAGE} +ARG AGENT_IMAGE=jenkins/inbound-agent:latest-jdk25 +FROM ${AGENT_IMAGE} USER root @@ -73,9 +73,10 @@ RUN wget -q https://github.com/restic/restic/releases/download/v${RESTIC}/restic COPY archive-restore.sh /usr/local/bin/archive-restore RUN chmod +x /usr/local/bin/archive-restore -# Restored stock builds and the restic cache live in the persistent -# jenkins_home volume, so each multi-GB build is downloaded once. -ENV DORKNET_CACHE=/var/jenkins_home/dorknet-cache +# Restored stock builds and the restic cache. Mount a volume here so each +# multi-GB stock build is downloaded once, not after every restart. +ENV DORKNET_CACHE=/home/jenkins/dorknet-cache +RUN mkdir -p ${DORKNET_CACHE} && chown jenkins:jenkins ${DORKNET_CACHE} ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 \ DOTNET_NOLOGO=1 diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md index a6f2eda..2b6b8e3 100644 --- a/ci/jenkins/README.md +++ b/ci/jenkins/README.md @@ -3,12 +3,13 @@ The `Jenkinsfile` at the repo root builds the DorkNet clients on your existing Jenkins. -It runs right on the Jenkins container, so that container's image needs the -build toolchain: Python with LIEF, the .NET SDK, the Android NDK and build-tools, -apktool, restic and rclone. `ci/jenkins/Dockerfile` is the stock -Jenkins image with that toolchain added. Your jobs, plugins and settings live -in the `jenkins_home` volume, not in the image, so switching images leaves -them alone. +The builds run on a **build agent**, not on Jenkins itself: +- **The agent:** a container built from `ci/jenkins/Dockerfile`, which is the + Jenkins inbound agent plus Python with LIEF, the .NET SDK, the Android NDK + and build-tools, apktool, restic and rclone. It runs on Coolify next to + Jenkins and connects to it over WebSocket on Docker's internal network. The + job runs on it through the `dorknet-builder` label. +- **Jenkins itself:** stays on the stock image and runs no builds. | Platform | Output (Jenkins artifacts) | How | |---|---|---| @@ -37,8 +38,8 @@ The job defaults to the builds this branch targets: Steam manifest build can be chosen per run. Tick **LIST_SNAPSHOTS** to print the snapshots matching a filter without building anything. -A restored snapshot is cached in `jenkins_home/dorknet-cache`, so each stock -build is downloaded once. +A restored snapshot is cached on the agent's `dorknet_cache` volume, so each +stock build is downloaded once. A Quest build needs a resolved hook table for its versionCode, committed under `DorkNet.Quest/targets/`. Only `20230317-2128210.json` exists today. The build @@ -57,32 +58,81 @@ Resolve a table for another build locally with `build.py build --dump-cs`. ## Setup -### 1. Switch your Jenkins to this image +### 1. Jenkins itself -On your Jenkins service in Dokploy: +The controller is `ci/jenkins/docker-compose.yml`: stock Jenkins, no build +tools. On Coolify, deploy it as a Docker Compose application from this +repository (base directory `/ci/jenkins`), or paste it into a Compose service. -1. **Build from this repo.** - - For an Application service: provider GitHub, branch `march-2023-03-21`, - build type Dockerfile, Dockerfile path `ci/jenkins/Dockerfile`, build - context `ci/jenkins`. - - For a Compose service: change its `jenkins` service to `build:` from - `ci/jenkins` as in `ci/jenkins/docker-compose.yml`. - - Either way, **keep the same volume on `/var/jenkins_home`**. -2. **Match the Jenkins version.** Set the build argument `JENKINS_IMAGE` to the - image the service runs today (e.g. `jenkins/jenkins:2.492.3-lts-jdk17`). - The default is `jenkins/jenkins:lts-jdk17`. -3. **Add the environment:** +- **Java 25.** The file defaults to `jenkins/jenkins:lts-jdk25`; set + `JENKINS_IMAGE` to pin a version, e.g. `jenkins/jenkins:2.587-jdk25`. + Jenkins supports Java 25 until September 2029, Java 21 only until September + 2027, and Java 17 not at all. Keep the agent on the same Java major version; + its image defaults to Java 25. +- **`restart: unless-stopped`** is in the file: a restart from the Jenkins UI + exits the container with code 5 and relies on Docker to start it again. +- **Turn on "Connect to predefined network"** (Advanced) so the agent can + reach Jenkins by its container name, `jenkins-`. +- **Run no builds on it:** **Manage Jenkins → Nodes → Built-In Node → + Configure**, set **Number of executors** to `0`. + +Earlier versions of this file built the controller from `ci/jenkins/Dockerfile`, +which is now the agent's image. The service and data volume keep their names, +so an existing Jenkins deployed from this file keeps its jobs and credentials +when redeployed; it switches to the stock image, and its builds wait until +the agent (step 3) is connected. + +That redeploy also upgrades Jenkins, and from Java 17 (the old default) to +Java 25, unless `JENKINS_IMAGE` pins your current image. Before it: + +1. **Back up the data volume** (`jenkins_home`), e.g. with a Coolify volume + backup or `docker run --rm -v :/v -v "$PWD":/b alpine tar czf + /b/jenkins_home.tgz -C /v .`, so you can go back to the old image. +2. After it starts, open **Manage Jenkins**. Plugins that need a newer + version show up there as failed to load; update them under **Plugins → + Updates** and restart. +3. Run the job once (**LIST_SNAPSHOTS** is a quick check) before relying on it. + +### 2. The agent node + +**Manage Jenkins → Nodes → New Node**, a permanent agent: + +| Setting | Value | +|---|---| +| Name | `dorknet-builder` | +| Labels | `dorknet-builder` | +| Remote root directory | `/home/jenkins/agent` | +| Launch method | Launch agent by connecting it to the controller, with **Use WebSocket** ticked | +| Number of executors | 1 | + +Save, open the node, and copy its **secret**. + +### 3. Deploy the agent on Coolify + +1. **Create a Docker Compose application** from this repository: build pack + Docker Compose, base directory **`/ci/jenkins`**, compose location + **`/docker-compose.agent.yml`**. The base directory is the build context, so with + `/` the build would pick up the server's `Dockerfile` at the repo root. It + needs no domain. Under Advanced, turn on **Connect to predefined network**. +2. **Set its environment:** | Variable | Value | |---|---| + | `JENKINS_URL` | Jenkins' **internal** address, `http://jenkins-:8080/`. Not the public URL, which is behind Cloudflare Access. | + | `JENKINS_SECRET` | the node secret from step 2 | + | `JENKINS_AGENT_NAME` | optional, default `dorknet-builder` | | `ARCHIVE_PC_REPO` | Steam repository folder on the FTP, e.g. `Rec Room Versions/steam-v2` (plain text, not URL-encoded) | | `ARCHIVE_QUEST_REPO` | Quest repository folder, e.g. `Rec Room Versions/quest-v2` | | `ARCHIVE_FTP_TLS` | `explicit` (default), `implicit` or `none` | -4. **Redeploy.** The first image build downloads the Android NDK (~1.5 GB) and - takes a while. +3. **Deploy.** The first image build downloads the Android NDK (~1.5 GB) and + takes a while. The node shows as connected in Jenkins once the agent is up. -### 2. Plugins +Both Jenkins and the agent join the server's predefined Coolify network, so +the agent reaches Jenkins by container name. Check the name under the Jenkins +service's containers before using it. + +### 4. Plugins The job needs these plugins: @@ -96,7 +146,7 @@ The job needs these plugins: Most are part of the suggested set. -### 3. Credentials +### 5. Credentials Add these under **Manage Jenkins → Credentials**, with exactly these IDs: @@ -115,7 +165,7 @@ Add these under **Manage Jenkins → Credentials**, with exactly these IDs: different key means every player has to uninstall before installing a CI build. -### 4. The job +### 6. The job 1. Create a **New Item → Pipeline** (e.g. `dorknet-client`). 2. Under **Pipeline**, choose **Pipeline script from SCM**: @@ -126,7 +176,7 @@ build. 4. Run it once with **Build** so Jenkins learns the parameters. Later runs offer **Build with Parameters**. -### 5. GitHub webhook +### 7. GitHub webhook In the repository's **Settings → Webhooks**: @@ -136,5 +186,5 @@ In the repository's **Settings → Webhooks**: ### Disk -The restored stock builds are cached in `jenkins_home/dorknet-cache`, several -GB each. Jenkins keeps the artifacts of the last 3 runs. +The agent's `dorknet_cache` volume holds the restored stock builds, several GB +each. Jenkins keeps the artifacts of the last 3 runs, on its own volume. diff --git a/ci/jenkins/docker-compose.agent.yml b/ci/jenkins/docker-compose.agent.yml new file mode 100644 index 0000000..1c9045c --- /dev/null +++ b/ci/jenkins/docker-compose.agent.yml @@ -0,0 +1,37 @@ +# DorkNet build agent on Coolify. Deploy as its own Docker Compose application +# from this repository with base directory /ci/jenkins, compose location +# /docker-compose.agent.yml and "Connect to predefined network" on. It needs no +# domain: it dials out to Jenkins over WebSocket on Docker's internal network. +# Set the variables in the resource's Environment Variables; see +# ci/jenkins/README.md. The controller is docker-compose.yml. +services: + agent: + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + # Reap the processes build tools leave behind (dotnet build servers, + # Gradle daemons, adb), as the inbound-agent image's docs ask for. + init: true + environment: + # Jenkins as seen from inside Docker (e.g. http://:8080), not the public URL, which is behind Cloudflare Access. + # Both are needed at runtime only; keep them out of the image build so + # the node secret never reaches a build argument. + JENKINS_URL: ${JENKINS_URL} + JENKINS_AGENT_NAME: ${JENKINS_AGENT_NAME:-dorknet-builder} + JENKINS_SECRET: ${JENKINS_SECRET} + JENKINS_WEB_SOCKET: "true" + JENKINS_AGENT_WORKDIR: /home/jenkins/agent + # Where the stock builds live in the archive (restic repository folders + # on the FTP, plain text). Overridable per run. + ARCHIVE_PC_REPO: ${ARCHIVE_PC_REPO:-} + ARCHIVE_QUEST_REPO: ${ARCHIVE_QUEST_REPO:-} + ARCHIVE_FTP_TLS: ${ARCHIVE_FTP_TLS:-explicit} + volumes: + - dorknet_cache:/home/jenkins/dorknet-cache + - agent_work:/home/jenkins/agent + +volumes: + dorknet_cache: + agent_work: diff --git a/ci/jenkins/docker-compose.yml b/ci/jenkins/docker-compose.yml index cc05993..f837683 100644 --- a/ci/jenkins/docker-compose.yml +++ b/ci/jenkins/docker-compose.yml @@ -1,29 +1,23 @@ -# Reference for the Jenkins service on Dokploy, built from this directory's -# Dockerfile (the stock Jenkins image plus the client build toolchain). +# The Jenkins controller: stock Jenkins on Java 25. It runs no builds; the +# client builds run on the agent (docker-compose.agent.yml). On Coolify, +# deploy it as a Docker Compose application from this repository (base +# directory /ci/jenkins) or paste it into a Compose service, with "Connect to +# predefined network" on so the agent can reach it. See ci/jenkins/README.md. # -# Use it on your EXISTING Jenkins service rather than as a second Jenkins. -# The data volume must be the one Jenkins uses today, or Jenkins starts empty. -# Point `volumes:` at that volume (and its name under the top-level -# `volumes:`, marked `external: true`), or keep this service's name and volume -# if they already match. See ci/jenkins/README.md. +# Earlier versions of this file built the controller from ci/jenkins/Dockerfile, +# which is now the agent's image. The service name and data volume are the +# same, so redeploying an existing Jenkins from this file keeps its jobs and +# credentials; its builds then wait for the agent. services: jenkins: - build: - context: . - dockerfile: Dockerfile - args: - # The image your Jenkins runs today, so the Jenkins version and Java - # stay the same. - JENKINS_IMAGE: ${JENKINS_IMAGE:-jenkins/jenkins:lts-jdk17} + # Pin a version with e.g. jenkins/jenkins:2.587-jdk25. Keep the agent on + # the same Java major version (its image defaults to Java 25). + image: ${JENKINS_IMAGE:-jenkins/jenkins:lts-jdk25} + # A restart from the Jenkins UI exits with code 5 and relies on Docker to + # start it again. restart: unless-stopped expose: - "8080" - environment: - # Where the stock builds live in the archive (restic repository folders - # on the FTP), as plain text. Overridable per run. - ARCHIVE_PC_REPO: ${ARCHIVE_PC_REPO:-} - ARCHIVE_QUEST_REPO: ${ARCHIVE_QUEST_REPO:-} - ARCHIVE_FTP_TLS: ${ARCHIVE_FTP_TLS:-explicit} volumes: - jenkins_home:/var/jenkins_home