diff --git a/.env.example b/.env.example index dada82f..dbee391 100644 --- a/.env.example +++ b/.env.example @@ -108,12 +108,21 @@ RECFLARE_DOMAIN=rec.example.com # RECFLARE_PHOTON_VOICE_APP_ID= # RECFLARE_PHOTON_CHAT_APP_ID= -# The Tachyon voice server (`match`, GET /player/connection-info): the `host:port` the -# client is handed as `voiceConnectionInfo`, and its id as `voiceServerId`. EMPTY unless -# you set them — no separate voice server. Set both or neither; like the Photon ids they -# are not secrets (the client receives them in the clear). +# The pool of Tachyon servers sessions are spread across (`match`, GET +# /player/connection-info): a COMMA-SEPARATED list of `host:port` entries, one of which +# the client is handed as `voiceConnectionInfo`. EMPTY unless you set it — no separate +# voice server. Not a secret (the client receives the address in the clear), like the +# Photon ids above. One entry is the ordinary single-server case: # RECFLARE_TACHYON_HOST_PORT=127.0.0.1:7777 -# RECFLARE_TACHYON_NAME=server-1 +# +# List several and each room instance is assigned one for its lifetime, so everybody in +# a session lands on the same server while different sessions spread across the pool. +# The id the client displays (`voiceServerId`) is generated from an entry's POSITION — +# `tachyon-1`, `tachyon-2`, … — so listing one address twice models two server slots on +# one box, and inserting an entry renames every server after it. The five below are a +# mock pool (RFC 5737 documentation addresses, which answer nothing): tachyon-1/-2 share +# a host, as do tachyon-4/-5. +# RECFLARE_TACHYON_HOST_PORT=198.51.100.10:7777,198.51.100.10:7778,198.51.100.11:7777,203.0.113.20:7777,203.0.113.20:7778 # The Photon region every session is pinned to (`match`). Unlike the app ids above this # does default, to `us` (us-east1) — an instance stamped with an empty region is one the diff --git a/apps/match/src/context.ts b/apps/match/src/context.ts index 1fd3dbf..72403e5 100644 --- a/apps/match/src/context.ts +++ b/apps/match/src/context.ts @@ -53,17 +53,21 @@ export type Env = SharedHonoEnv & { /** The Photon Chat application id. Optional; see {@link Env.PHOTON_REALTIME_APP_ID}. */ PHOTON_CHAT_APP_ID?: string /** - * The Tachyon voice server the client is handed as `voiceConnectionInfo` on - * `GET /player/connection-info`, as `host:port` (e.g. `66.228.47.217:7777`). - * Optional, and EMPTY when unset — no separate voice server. Not a secret (the - * client receives it in the clear), so a plain var like the Photon ids. + * The pool of Tachyon servers sessions are spread across — a COMMA-SEPARATED list of + * `host:port` entries (e.g. `66.228.47.217:7777,66.228.47.217:7778,45.79.2.10:7777`). + * One entry is a single server, which is the common case. Optional, and EMPTY when + * unset — no separate voice server, and the connection info's voice fields stay empty. + * Not a secret (the client receives it in the clear), so a plain var like the Photon + * ids. + * + * A room instance is assigned one entry for its lifetime and every player in it is + * handed that one, derived from the instance id rather than stored — see + * `tachyonServerFor` in match.app.ts, which also explains what changing this list does + * to sessions already running. The `voiceServerId` the client displays is GENERATED + * from an entry's position (`tachyon-1`, `tachyon-2`, …), so the same address may be + * listed twice to model two server slots on one box. */ TACHYON_HOST_PORT?: string - /** - * The id of that voice server, handed to the client as `voiceServerId` (e.g. - * `server-1`). Optional; see {@link Env.TACHYON_HOST_PORT} — set both or neither. - */ - TACHYON_NAME?: string /** * The Photon region every session is pinned to — both the region named in the connection * info and the one stamped on every room instance, which must agree. Optional; unlike the diff --git a/apps/match/src/match.app.ts b/apps/match/src/match.app.ts index 535fa92..e5bdf94 100644 --- a/apps/match/src/match.app.ts +++ b/apps/match/src/match.app.ts @@ -166,6 +166,66 @@ function instancePhotonRegion(env: Env): string { return varOr(env.PHOTON_REGION, DEFAULT_PHOTON_REGION) } +/** + * One Tachyon server this deployment can put a session on: where the client connects + * (`host:port`) and the cosmetic id it displays for it. + */ +interface TachyonServer { + hostPort: string + serverId: string +} + +/** + * The Tachyon servers sessions are spread across, from `TACHYON_HOST_PORT` — a + * comma-separated list of `host:port` entries. EMPTY when the var is unset: recflare + * runs no Tachyon server of its own, and a client handed an address that answers + * nothing is worse off than one told there is no voice server at all. + * + * An entry's POSITION in the list is its identity: `voiceServerId` is generated from it + * (`tachyon-1`, `tachyon-2`, …) rather than configured, so the same host may appear + * twice and count as two servers — which is what one box running several server slots + * looks like from here. The id is cosmetic (the client connects to the address and + * never sends the id anywhere), but it is positional, so inserting an entry renames + * every server after it. + */ +function tachyonPool(env: Env): TachyonServer[] { + return varOr(env.TACHYON_HOST_PORT, '') + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => entry !== '') + .map((hostPort, i) => ({ hostPort, serverId: `tachyon-${i + 1}` })) +} + +/** What the connection info carries when there is no Tachyon server to name. */ +const NO_TACHYON_SERVER: TachyonServer = { hostPort: '', serverId: '' } + +/** + * The Tachyon server a room instance runs on — the whole of the distributed selection, + * and deliberately a pure function of the instance id rather than a stored assignment. + * + * The point of a server assignment is that everyone in one instance is handed the SAME + * one: the player whose matchmake created the instance and everyone who joins it later + * each call `GET /player/connection-info` separately, so an assignment made per REQUEST + * (random, round-robin over a counter, least-loaded) would scatter one session across + * the pool. Deriving it from the instance id instead makes every caller compute the same + * answer without coordinating, needs no column to persist and no cleanup when the + * instance is swept, and answers for instances created before this existed. + * + * Instance ids are sequential ({@link createRoomInstance} allocates `MAX(id) + 1`), so + * the modulo hands successive instances to successive servers: a plain round-robin over + * instances, which is the spread a real allocator would aim for anyway. Changing the + * pool DOES move live instances — the list is the assignment — so add entries to the + * end and expect a session mid-flight to be told a different server when you don't. + * + * `roomInstanceId` 0 means the caller resolved to no instance at all (they're in no + * room, or named one that doesn't exist); they get no server rather than server one. + */ +function tachyonServerFor(env: Env, roomInstanceId: number): TachyonServer { + const pool = tachyonPool(env) + if (pool.length === 0 || roomInstanceId <= 0) return NO_TACHYON_SERVER + return pool[roomInstanceId % pool.length] ?? NO_TACHYON_SERVER +} + /** * Networking feature flags the client reads off its connection info. Verbatim from * the reference server — the client changes how it replicates based on these, so they @@ -2693,6 +2753,10 @@ const app = new Hono() // reads presence and nothing else; we fall back to looking the `roomInstanceId` query // param up when presence has no room (it expires on a TTL, and the client sometimes // asks before matchmaking has landed), and to an empty string when neither resolves. + // + // The Tachyon server is resolved from the same instance ({@link tachyonServerFor}), so + // the player who created the session and everyone who joins it later are all sent to + // one server without this endpoint having to remember what it told the first caller. .get( '/player/connection-info', describeRoute({ @@ -2703,9 +2767,11 @@ const app = new Hono() '`{ success, value, error }` envelope: a freshly minted `photonAuthToken`, the', 'Photon application ids, and the `photonRoomId` of the instance the caller is in', '(from their presence, falling back to the `roomInstanceId` query param). The voice', - 'fields carry the Tachyon voice server (`TACHYON_HOST_PORT`/`TACHYON_NAME`),', - 'empty when none is configured. `experiments` carries the', - 'client’s networking flags.', + 'fields name the Tachyon server that instance was assigned — one entry out of the', + '`TACHYON_HOST_PORT` pool, chosen by instance id so every player in a session is', + 'handed the same one, with a generated `voiceServerId` (`tachyon-1`, `tachyon-2`,', + '…). Both are empty when the pool is unset or the caller is in no instance.', + '`experiments` carries the client’s networking flags.', ].join(' '), security: AUTHED, parameters: [ @@ -2729,14 +2795,22 @@ const app = new Hono() const apps = photonApps(c.env) const presence = await getPresence(c.env.DB, id) // Presence first (it's the instance the player is actually in); the query param - // only stands in when there's no live presence to read. + // only stands in when there's no live presence to read. The instance id travels + // with the Photon room because the Tachyon server is derived from it — resolving + // one without the other would hand a joiner the right Photon room on a different + // game server than the rest of their session. + let roomInstanceId = presence?.roomInstance?.roomInstanceId ?? 0 let photonRoomId = presence?.roomInstance?.photonRoomId ?? '' if (!photonRoomId) { const requested = Number.parseInt(c.req.query('roomInstanceId') ?? '', 10) if (!Number.isNaN(requested)) { - photonRoomId = (await getRoomInstance(c.env.DB, requested))?.photonRoomId ?? '' + const instance = await getRoomInstance(c.env.DB, requested) + roomInstanceId = instance?.roomInstanceId ?? 0 + photonRoomId = instance?.photonRoomId ?? '' } } + // The instance's server, the same one every other player in it is handed. + const tachyon = tachyonServerFor(c.env, roomInstanceId) // Identifies the player to Photon. Signed with the shared JWT secret; the token's // `aud` is the realtime app it's for. Nothing verifies it while Photon is @@ -2758,13 +2832,14 @@ const app = new Hono() photonAuthToken, ...apps, photonRoomId, - // The Tachyon voice server, from the operator's vars — empty strings when - // unset (no separate voice server). Empty rather than null: the client's + // The Tachyon server this instance runs on, picked out of the operator's + // pool by {@link tachyonServerFor} — empty strings when the pool is empty + // or the caller is in no instance. Empty rather than null: the client's // decoder is likelier to accept a missing-value string than a null on a // string field. The presence payload's NULL_CONNECTION_INFO keeps its // nulls — that one never carries credentials. - voiceConnectionInfo: varOr(c.env.TACHYON_HOST_PORT, ''), - voiceServerId: varOr(c.env.TACHYON_NAME, ''), + voiceConnectionInfo: tachyon.hostPort, + voiceServerId: tachyon.serverId, experiments: PHOTON_EXPERIMENTS, }, error: null, diff --git a/apps/match/src/openapi.ts b/apps/match/src/openapi.ts index 2885583..471cc8d 100644 --- a/apps/match/src/openapi.ts +++ b/apps/match/src/openapi.ts @@ -206,8 +206,9 @@ export const ConnectionExperiments = z.object({ * recflare; what varies per caller is `photonAuthToken` (minted for them on the spot) * and `photonRoomId`, the Photon room of the instance their presence says they're in * — the same name every other player in that instance is handed. The voice fields name - * the Tachyon voice server (`TACHYON_HOST_PORT`/`TACHYON_NAME` vars), empty when none - * is configured. `photonRegion` matches the one stamped + * the Tachyon server that instance was assigned out of the `TACHYON_HOST_PORT` pool — + * likewise the same for everyone in the session — and are empty when the pool is unset + * or the caller is in no instance. `photonRegion` matches the one stamped * on every room instance, so the two can't disagree. */ export const ConnectionInfo = z.object({ @@ -219,8 +220,10 @@ export const ConnectionInfo = z.object({ photonRoomId: z.string().describe('The caller’s current instance; empty when they’re in none'), voiceConnectionInfo: z .string() - .describe('The Tachyon voice server, `host:port`; empty when none is configured'), - voiceServerId: z.string().describe('The Tachyon voice server id; empty when none is configured'), + .describe('The instance’s Tachyon server, `host:port`; empty when none is configured'), + voiceServerId: z + .string() + .describe('That server’s generated id (`tachyon-1`, …); cosmetic, empty when there is none'), experiments: ConnectionExperiments, }) diff --git a/apps/match/src/test/integration/api.test.ts b/apps/match/src/test/integration/api.test.ts index 191bb5a..0677334 100644 --- a/apps/match/src/test/integration/api.test.ts +++ b/apps/match/src/test/integration/api.test.ts @@ -1247,8 +1247,9 @@ describe('auth-gated endpoints', () => { // The room the client is told to join has to be the one matchmaking placed // them in, or they end up alone in a room of their own. photonRoomId: matchmaked.roomInstance.photonRoomId, - // Empty strings, not nulls — unlike the presence payload's connection fields, - // which stay null (they never carry credentials). + // No TACHYON_HOST_PORT pool configured, so there is no server to name. Empty + // strings, not nulls — unlike the presence payload's connection fields, which + // stay null (they never carry credentials). voiceConnectionInfo: '', voiceServerId: '', experiments: { @@ -1377,6 +1378,84 @@ describe('auth-gated endpoints', () => { expect(body.value.photonRoomId).toBe('') }) + test('the Tachyon server is assigned per room instance, not per request', async () => { + // tachyon-1/-2 and tachyon-4/-5 are slots on one host apiece: a server id names a + // slot, which is why it's generated from the entry's position and not from its + // address. The blank entry and the stray spaces below are dropped — a list edited + // by hand shouldn't hand anyone an empty address. + const pool = [ + { voiceConnectionInfo: '198.51.100.10:7777', voiceServerId: 'tachyon-1' }, + { voiceConnectionInfo: '198.51.100.10:7778', voiceServerId: 'tachyon-2' }, + { voiceConnectionInfo: '198.51.100.11:7777', voiceServerId: 'tachyon-3' }, + { voiceConnectionInfo: '203.0.113.20:7777', voiceServerId: 'tachyon-4' }, + { voiceConnectionInfo: '203.0.113.20:7778', voiceServerId: 'tachyon-5' }, + ] + const original = env.TACHYON_HOST_PORT + try { + env.TACHYON_HOST_PORT = + '198.51.100.10:7777, 198.51.100.10:7778 ,,198.51.100.11:7777,203.0.113.20:7777,203.0.113.20:7778' + + const voiceFor = async (player: string, roomInstanceId?: number) => { + const res = await exports.default.fetch( + roomInstanceId === undefined + ? `${ORIGIN}/player/connection-info` + : `${ORIGIN}/player/connection-info?roomInstanceId=${roomInstanceId}`, + { headers: await bearer(player) } + ) + const body = (await res.json()) as { + value: { voiceConnectionInfo: string; voiceServerId: string } + } + return { + voiceConnectionInfo: body.value.voiceConnectionInfo, + voiceServerId: body.value.voiceServerId, + } + } + + // The player who opened the session reads their server off their presence... + const instance = await createRoomInstance(env.DB, { + ownerAccountId: 970, + roomId: 2, + photonRoomId: 'tachyon-instance-a', + maxCapacity: 12, + }) + await setPresence(env.DB, { + accountId: 970, + roomInstance: instance, + statusVisibility: 0, + deviceClass: 0, + vrMovementMode: 1, + platform: 0, + appVersion: GAME_VERSION, + }) + const assigned = pool[instance.roomInstanceId % pool.length]! + expect(await voiceFor('970')).toEqual(assigned) + + // ...and a joiner asking by instance id, before their own presence has landed, + // is sent to the same one. Two players in a session on two servers is the whole + // failure this is arranged to avoid. + expect(await voiceFor('971', instance.roomInstanceId)).toEqual(assigned) + + // The next session opened goes to the next server along — instance ids are + // sequential, so the pool is walked round-robin as instances are created. + const next = await createRoomInstance(env.DB, { + ownerAccountId: 972, + roomId: 2, + photonRoomId: 'tachyon-instance-b', + maxCapacity: 12, + }) + expect(next.roomInstanceId).toBe(instance.roomInstanceId + 1) + const alongside = await voiceFor('972', next.roomInstanceId) + expect(alongside).toEqual(pool[next.roomInstanceId % pool.length]) + expect(alongside.voiceServerId).not.toBe(assigned.voiceServerId) + + // A player in no instance gets no server, pool or no pool — there is nothing for + // them to be on the same server as, and the fields stay empty strings. + expect(await voiceFor('973')).toEqual({ voiceConnectionInfo: '', voiceServerId: '' }) + } finally { + env.TACHYON_HOST_PORT = original + } + }) + test('re-matchmaking into your current room returns a different instance (id must change)', async () => { // The client keys the room transition off a changing roomInstanceId; handing back // the instance the player is already in hangs their join. RecCenter (cap 12) so @@ -3073,9 +3152,7 @@ describe('auth-gated endpoints', () => { // Unauthenticated is a 401, not a refusal code. expect( - ( - await exports.default.fetch(`${ORIGIN}/matchmake/v2/player/8811`, { method: 'POST' }) - ).status + (await exports.default.fetch(`${ORIGIN}/matchmake/v2/player/8811`, { method: 'POST' })).status ).toBe(401) }) diff --git a/apps/match/wrangler.jsonc b/apps/match/wrangler.jsonc index a6a9d19..948660b 100644 --- a/apps/match/wrangler.jsonc +++ b/apps/match/wrangler.jsonc @@ -68,7 +68,7 @@ }, // The operator's knobs — the room substitutions (ROOM_REDIRECTS), the Photon app ids // and region (PHOTON_REALTIME_APP_ID, PHOTON_VOICE_APP_ID, PHOTON_CHAT_APP_ID, - // PHOTON_REGION), and the Tachyon voice server (TACHYON_HOST_PORT, TACHYON_NAME) — + // PHOTON_REGION), and the Tachyon server pool (TACHYON_HOST_PORT) — // are deliberately NOT set here. They're injected at deploy time from // the gitignored .env (RECFLARE_, see .env.example), so swapping a room out or // pointing at your own Photon apps never means editing a versioned file. Unset — the