[match] support a cluster of tachyon servers

.env.example

@@ -108,12 +108,21 @@ RECFLARE_DOMAIN=rec.example.com
 # RECFLARE_PHOTON_VOICE_APP_ID=
 # RECFLARE_PHOTON_CHAT_APP_ID=
 
-# The Tachyon voice server (`match`, GET /player/connection-info): the `host:port` the
+# The pool of Tachyon servers sessions are spread across (`match`, GET
-# client is handed as `voiceConnectionInfo`, and its id as `voiceServerId`. EMPTY unless
+# /player/connection-info): a COMMA-SEPARATED list of `host:port` entries, one of which
-# you set them — no separate voice server. Set both or neither; like the Photon ids they
+# the client is handed as `voiceConnectionInfo`. EMPTY unless you set it — no separate
-# are not secrets (the client receives them in the clear).
+# voice server. Not a secret (the client receives the address in the clear), like the
+# Photon ids above. One entry is the ordinary single-server case:
 # RECFLARE_TACHYON_HOST_PORT=127.0.0.1:7777
-# RECFLARE_TACHYON_NAME=server-1
+#
+# List several and each room instance is assigned one for its lifetime, so everybody in
+# a session lands on the same server while different sessions spread across the pool.
+# The id the client displays (`voiceServerId`) is generated from an entry's POSITION —
+# `tachyon-1`, `tachyon-2`, … — so listing one address twice models two server slots on
+# one box, and inserting an entry renames every server after it. The five below are a
+# mock pool (RFC 5737 documentation addresses, which answer nothing): tachyon-1/-2 share
+# a host, as do tachyon-4/-5.
+# RECFLARE_TACHYON_HOST_PORT=198.51.100.10:7777,198.51.100.10:7778,198.51.100.11:7777,203.0.113.20:7777,203.0.113.20:7778
 
 # The Photon region every session is pinned to (`match`). Unlike the app ids above this
 # does default, to `us` (us-east1) — an instance stamped with an empty region is one the

apps/match/src/context.ts

@@ -53,17 +53,21 @@ export type Env = SharedHonoEnv & {
 	/** The Photon Chat application id. Optional; see {@link Env.PHOTON_REALTIME_APP_ID}. */
 	PHOTON_CHAT_APP_ID?: string
 	/**
-	 * The Tachyon voice server the client is handed as `voiceConnectionInfo` on
+	 * The pool of Tachyon servers sessions are spread across — a COMMA-SEPARATED list of
-	 * `GET /player/connection-info`, as `host:port` (e.g. `66.228.47.217:7777`).
+	 * `host:port` entries (e.g. `66.228.47.217:7777,66.228.47.217:7778,45.79.2.10:7777`).
-	 * Optional, and EMPTY when unset — no separate voice server. Not a secret (the
+	 * One entry is a single server, which is the common case. Optional, and EMPTY when
-	 * client receives it in the clear), so a plain var like the Photon ids.
+	 * unset — no separate voice server, and the connection info's voice fields stay empty.
+	 * Not a secret (the client receives it in the clear), so a plain var like the Photon
+	 * ids.
+	 *
+	 * A room instance is assigned one entry for its lifetime and every player in it is
+	 * handed that one, derived from the instance id rather than stored — see
+	 * `tachyonServerFor` in match.app.ts, which also explains what changing this list does
+	 * to sessions already running. The `voiceServerId` the client displays is GENERATED
+	 * from an entry's position (`tachyon-1`, `tachyon-2`, …), so the same address may be
+	 * listed twice to model two server slots on one box.
 	 */
 	TACHYON_HOST_PORT?: string
-	/**
-	 * The id of that voice server, handed to the client as `voiceServerId` (e.g.
-	 * `server-1`). Optional; see {@link Env.TACHYON_HOST_PORT} — set both or neither.
-	 */
-	TACHYON_NAME?: string
 	/**
 	 * The Photon region every session is pinned to — both the region named in the connection
 	 * info and the one stamped on every room instance, which must agree. Optional; unlike the

apps/match/src/match.app.ts

@@ -166,6 +166,66 @@ function instancePhotonRegion(env: Env): string {
 	return varOr(env.PHOTON_REGION, DEFAULT_PHOTON_REGION)
 }
 
+/**
+ * One Tachyon server this deployment can put a session on: where the client connects
+ * (`host:port`) and the cosmetic id it displays for it.
+ */
+interface TachyonServer {
+	hostPort: string
+	serverId: string
+}
+
+/**
+ * The Tachyon servers sessions are spread across, from `TACHYON_HOST_PORT` — a
+ * comma-separated list of `host:port` entries. EMPTY when the var is unset: recflare
+ * runs no Tachyon server of its own, and a client handed an address that answers
+ * nothing is worse off than one told there is no voice server at all.
+ *
+ * An entry's POSITION in the list is its identity: `voiceServerId` is generated from it
+ * (`tachyon-1`, `tachyon-2`, …) rather than configured, so the same host may appear
+ * twice and count as two servers — which is what one box running several server slots
+ * looks like from here. The id is cosmetic (the client connects to the address and
+ * never sends the id anywhere), but it is positional, so inserting an entry renames
+ * every server after it.
+ */
+function tachyonPool(env: Env): TachyonServer[] {
+	return varOr(env.TACHYON_HOST_PORT, '')
+		.split(',')
+		.map((entry) => entry.trim())
+		.filter((entry) => entry !== '')
+		.map((hostPort, i) => ({ hostPort, serverId: `tachyon-${i + 1}` }))
+}
+
+/** What the connection info carries when there is no Tachyon server to name. */
+const NO_TACHYON_SERVER: TachyonServer = { hostPort: '', serverId: '' }
+
+/**
+ * The Tachyon server a room instance runs on — the whole of the distributed selection,
+ * and deliberately a pure function of the instance id rather than a stored assignment.
+ *
+ * The point of a server assignment is that everyone in one instance is handed the SAME
+ * one: the player whose matchmake created the instance and everyone who joins it later
+ * each call `GET /player/connection-info` separately, so an assignment made per REQUEST
+ * (random, round-robin over a counter, least-loaded) would scatter one session across
+ * the pool. Deriving it from the instance id instead makes every caller compute the same
+ * answer without coordinating, needs no column to persist and no cleanup when the
+ * instance is swept, and answers for instances created before this existed.
+ *
+ * Instance ids are sequential ({@link createRoomInstance} allocates `MAX(id) + 1`), so
+ * the modulo hands successive instances to successive servers: a plain round-robin over
+ * instances, which is the spread a real allocator would aim for anyway. Changing the
+ * pool DOES move live instances — the list is the assignment — so add entries to the
+ * end and expect a session mid-flight to be told a different server when you don't.
+ *
+ * `roomInstanceId` 0 means the caller resolved to no instance at all (they're in no
+ * room, or named one that doesn't exist); they get no server rather than server one.
+ */
+function tachyonServerFor(env: Env, roomInstanceId: number): TachyonServer {
+	const pool = tachyonPool(env)
+	if (pool.length === 0 || roomInstanceId <= 0) return NO_TACHYON_SERVER
+	return pool[roomInstanceId % pool.length] ?? NO_TACHYON_SERVER
+}
+
 /**
  * Networking feature flags the client reads off its connection info. Verbatim from
  * the reference server — the client changes how it replicates based on these, so they
@@ -2693,6 +2753,10 @@ const app = new Hono<App>()
 	// reads presence and nothing else; we fall back to looking the `roomInstanceId` query
 	// param up when presence has no room (it expires on a TTL, and the client sometimes
 	// asks before matchmaking has landed), and to an empty string when neither resolves.
+	//
+	// The Tachyon server is resolved from the same instance ({@link tachyonServerFor}), so
+	// the player who created the session and everyone who joins it later are all sent to
+	// one server without this endpoint having to remember what it told the first caller.
 	.get(
 		'/player/connection-info',
 		describeRoute({
@@ -2703,9 +2767,11 @@ const app = new Hono<App>()
 				'`{ success, value, error }` envelope: a freshly minted `photonAuthToken`, the',
 				'Photon application ids, and the `photonRoomId` of the instance the caller is in',
 				'(from their presence, falling back to the `roomInstanceId` query param). The voice',
-				'fields carry the Tachyon voice server (`TACHYON_HOST_PORT`/`TACHYON_NAME`),',
+				'fields name the Tachyon server that instance was assigned — one entry out of the',
-				'empty when none is configured. `experiments` carries the',
+				'`TACHYON_HOST_PORT` pool, chosen by instance id so every player in a session is',
-				'client’s networking flags.',
+				'handed the same one, with a generated `voiceServerId` (`tachyon-1`, `tachyon-2`,',
+				'…). Both are empty when the pool is unset or the caller is in no instance.',
+				'`experiments` carries the client’s networking flags.',
 			].join(' '),
 			security: AUTHED,
 			parameters: [
@@ -2729,14 +2795,22 @@ const app = new Hono<App>()
 			const apps = photonApps(c.env)
 			const presence = await getPresence<RoomInstance>(c.env.DB, id)
 			// Presence first (it's the instance the player is actually in); the query param
-			// only stands in when there's no live presence to read.
+			// only stands in when there's no live presence to read. The instance id travels
+			// with the Photon room because the Tachyon server is derived from it — resolving
+			// one without the other would hand a joiner the right Photon room on a different
+			// game server than the rest of their session.
+			let roomInstanceId = presence?.roomInstance?.roomInstanceId ?? 0
 			let photonRoomId = presence?.roomInstance?.photonRoomId ?? ''
 			if (!photonRoomId) {
 				const requested = Number.parseInt(c.req.query('roomInstanceId') ?? '', 10)
 				if (!Number.isNaN(requested)) {
-					photonRoomId = (await getRoomInstance(c.env.DB, requested))?.photonRoomId ?? ''
+					const instance = await getRoomInstance(c.env.DB, requested)
+					roomInstanceId = instance?.roomInstanceId ?? 0
+					photonRoomId = instance?.photonRoomId ?? ''
 				}
 			}
+			// The instance's server, the same one every other player in it is handed.
+			const tachyon = tachyonServerFor(c.env, roomInstanceId)
 
 			// Identifies the player to Photon. Signed with the shared JWT secret; the token's
 			// `aud` is the realtime app it's for. Nothing verifies it while Photon is
@@ -2758,13 +2832,14 @@ const app = new Hono<App>()
 					photonAuthToken,
 					...apps,
 					photonRoomId,
-					// The Tachyon voice server, from the operator's vars — empty strings when
+					// The Tachyon server this instance runs on, picked out of the operator's
-					// unset (no separate voice server). Empty rather than null: the client's
+					// pool by {@link tachyonServerFor} — empty strings when the pool is empty
+					// or the caller is in no instance. Empty rather than null: the client's
 					// decoder is likelier to accept a missing-value string than a null on a
 					// string field. The presence payload's NULL_CONNECTION_INFO keeps its
 					// nulls — that one never carries credentials.
-					voiceConnectionInfo: varOr(c.env.TACHYON_HOST_PORT, ''),
+					voiceConnectionInfo: tachyon.hostPort,
-					voiceServerId: varOr(c.env.TACHYON_NAME, ''),
+					voiceServerId: tachyon.serverId,
 					experiments: PHOTON_EXPERIMENTS,
 				},
 				error: null,

apps/match/src/openapi.ts

@@ -206,8 +206,9 @@ export const ConnectionExperiments = z.object({
  * recflare; what varies per caller is `photonAuthToken` (minted for them on the spot)
  * and `photonRoomId`, the Photon room of the instance their presence says they're in
  * — the same name every other player in that instance is handed. The voice fields name
- * the Tachyon voice server (`TACHYON_HOST_PORT`/`TACHYON_NAME` vars), empty when none
+ * the Tachyon server that instance was assigned out of the `TACHYON_HOST_PORT` pool —
- * is configured. `photonRegion` matches the one stamped
+ * likewise the same for everyone in the session — and are empty when the pool is unset
+ * or the caller is in no instance. `photonRegion` matches the one stamped
  * on every room instance, so the two can't disagree.
  */
 export const ConnectionInfo = z.object({
@@ -219,8 +220,10 @@ export const ConnectionInfo = z.object({
 	photonRoomId: z.string().describe('The caller’s current instance; empty when they’re in none'),
 	voiceConnectionInfo: z
 		.string()
-		.describe('The Tachyon voice server, `host:port`; empty when none is configured'),
+		.describe('The instance’s Tachyon server, `host:port`; empty when none is configured'),
-	voiceServerId: z.string().describe('The Tachyon voice server id; empty when none is configured'),
+	voiceServerId: z
+		.string()
+		.describe('That server’s generated id (`tachyon-1`, …); cosmetic, empty when there is none'),
 	experiments: ConnectionExperiments,
 })
 

apps/match/src/test/integration/api.test.ts

@@ -1247,8 +1247,9 @@ describe('auth-gated endpoints', () => {
 				// The room the client is told to join has to be the one matchmaking placed
 				// them in, or they end up alone in a room of their own.
 				photonRoomId: matchmaked.roomInstance.photonRoomId,
-				// Empty strings, not nulls — unlike the presence payload's connection fields,
+				// No TACHYON_HOST_PORT pool configured, so there is no server to name. Empty
-				// which stay null (they never carry credentials).
+				// strings, not nulls — unlike the presence payload's connection fields, which
+				// stay null (they never carry credentials).
 				voiceConnectionInfo: '',
 				voiceServerId: '',
 				experiments: {
@@ -1377,6 +1378,84 @@ describe('auth-gated endpoints', () => {
 		expect(body.value.photonRoomId).toBe('')
 	})
 
+	test('the Tachyon server is assigned per room instance, not per request', async () => {
+		// tachyon-1/-2 and tachyon-4/-5 are slots on one host apiece: a server id names a
+		// slot, which is why it's generated from the entry's position and not from its
+		// address. The blank entry and the stray spaces below are dropped — a list edited
+		// by hand shouldn't hand anyone an empty address.
+		const pool = [
+			{ voiceConnectionInfo: '198.51.100.10:7777', voiceServerId: 'tachyon-1' },
+			{ voiceConnectionInfo: '198.51.100.10:7778', voiceServerId: 'tachyon-2' },
+			{ voiceConnectionInfo: '198.51.100.11:7777', voiceServerId: 'tachyon-3' },
+			{ voiceConnectionInfo: '203.0.113.20:7777', voiceServerId: 'tachyon-4' },
+			{ voiceConnectionInfo: '203.0.113.20:7778', voiceServerId: 'tachyon-5' },
+		]
+		const original = env.TACHYON_HOST_PORT
+		try {
+			env.TACHYON_HOST_PORT =
+				'198.51.100.10:7777, 198.51.100.10:7778 ,,198.51.100.11:7777,203.0.113.20:7777,203.0.113.20:7778'
+
+			const voiceFor = async (player: string, roomInstanceId?: number) => {
+				const res = await exports.default.fetch(
+					roomInstanceId === undefined
+						? `${ORIGIN}/player/connection-info`
+						: `${ORIGIN}/player/connection-info?roomInstanceId=${roomInstanceId}`,
+					{ headers: await bearer(player) }
+				)
+				const body = (await res.json()) as {
+					value: { voiceConnectionInfo: string; voiceServerId: string }
+				}
+				return {
+					voiceConnectionInfo: body.value.voiceConnectionInfo,
+					voiceServerId: body.value.voiceServerId,
+				}
+			}
+
+			// The player who opened the session reads their server off their presence...
+			const instance = await createRoomInstance(env.DB, {
+				ownerAccountId: 970,
+				roomId: 2,
+				photonRoomId: 'tachyon-instance-a',
+				maxCapacity: 12,
+			})
+			await setPresence(env.DB, {
+				accountId: 970,
+				roomInstance: instance,
+				statusVisibility: 0,
+				deviceClass: 0,
+				vrMovementMode: 1,
+				platform: 0,
+				appVersion: GAME_VERSION,
+			})
+			const assigned = pool[instance.roomInstanceId % pool.length]!
+			expect(await voiceFor('970')).toEqual(assigned)
+
+			// ...and a joiner asking by instance id, before their own presence has landed,
+			// is sent to the same one. Two players in a session on two servers is the whole
+			// failure this is arranged to avoid.
+			expect(await voiceFor('971', instance.roomInstanceId)).toEqual(assigned)
+
+			// The next session opened goes to the next server along — instance ids are
+			// sequential, so the pool is walked round-robin as instances are created.
+			const next = await createRoomInstance(env.DB, {
+				ownerAccountId: 972,
+				roomId: 2,
+				photonRoomId: 'tachyon-instance-b',
+				maxCapacity: 12,
+			})
+			expect(next.roomInstanceId).toBe(instance.roomInstanceId + 1)
+			const alongside = await voiceFor('972', next.roomInstanceId)
+			expect(alongside).toEqual(pool[next.roomInstanceId % pool.length])
+			expect(alongside.voiceServerId).not.toBe(assigned.voiceServerId)
+
+			// A player in no instance gets no server, pool or no pool — there is nothing for
+			// them to be on the same server as, and the fields stay empty strings.
+			expect(await voiceFor('973')).toEqual({ voiceConnectionInfo: '', voiceServerId: '' })
+		} finally {
+			env.TACHYON_HOST_PORT = original
+		}
+	})
+
 	test('re-matchmaking into your current room returns a different instance (id must change)', async () => {
 		// The client keys the room transition off a changing roomInstanceId; handing back
 		// the instance the player is already in hangs their join. RecCenter (cap 12) so
@@ -3073,9 +3152,7 @@ describe('auth-gated endpoints', () => {
 
 		// Unauthenticated is a 401, not a refusal code.
 		expect(
-			(
+			(await exports.default.fetch(`${ORIGIN}/matchmake/v2/player/8811`, { method: 'POST' })).status
-				await exports.default.fetch(`${ORIGIN}/matchmake/v2/player/8811`, { method: 'POST' })
-			).status
 		).toBe(401)
 	})
 

apps/match/wrangler.jsonc

@@ -68,7 +68,7 @@
 	},
 	// The operator's knobs — the room substitutions (ROOM_REDIRECTS), the Photon app ids
 	// and region (PHOTON_REALTIME_APP_ID, PHOTON_VOICE_APP_ID, PHOTON_CHAT_APP_ID,
-	// PHOTON_REGION), and the Tachyon voice server (TACHYON_HOST_PORT, TACHYON_NAME) —
+	// PHOTON_REGION), and the Tachyon server pool (TACHYON_HOST_PORT) —
 	// are deliberately NOT set here. They're injected at deploy time from
 	// the gitignored .env (RECFLARE_<VAR>, see .env.example), so swapping a room out or
 	// pointing at your own Photon apps never means editing a versioned file. Unset — the