[www] token drop performance improvement

apps/econ/src/balance-db.ts

@@ -1,6 +1,8 @@
 import { BalancePlatform } from '../../notify/src/notification-payloads'
 import { CurrencyType } from './currency'
 
+import type { GiftContent } from '@repo/domain'
+
 // The currency vocabulary and the Plus reload SQL live in the import-free `currency.ts` so
 // the Node CLI in @repo/tools can share them; re-exported here so this stays the module
 // everything else reads balances from.
@@ -199,3 +201,136 @@ export async function spendCurrency(
 		.run()
 	return meta.changes > 0
 }
+
+/** One token box to hand over: whose, how many, and the stored box. See {@link payTokenGifts}. */
+export interface TokenGift {
+	accountId: number
+	/** Signed: positive credits, negative debits (guarded — see below), zero only boxes. */
+	amount: number
+	/** The box as stored in `received_gift`, serialized as-is. */
+	content: GiftContent
+}
+
+/** One box handed over by {@link payTokenGifts}: the resulting balance and the box's id. */
+export interface PaidTokenGift extends TokenGift {
+	balance: number
+	giftId: number
+}
+
+/**
+ * How many accounts one round of {@link payTokenGifts} writes at once. Every statement
+ * binds three values per account, and D1 allows 100 bound parameters a statement.
+ */
+export const TOKEN_GIFT_CHUNK = 30
+
+/**
+ * Hand RecCenterTokens in a gift box to MANY accounts at once — the staff drops, the room
+ * gift and the Discord supporter cron, which all pay a crowd. Paid one account at a time
+ * (seed, credit, re-read, box: five D1 round trips each) a full server was ~1,250 queries
+ * in series, past D1's 1,000-queries-an-invocation cap somewhere around 200 players, and
+ * died partway with the first players paid and the rest not. This pays
+ * {@link TOKEN_GIFT_CHUNK} accounts a round in two batches of multi-row statements.
+ *
+ * Per account it is exactly the one-player grant: the signup grant seeded first (a
+ * never-touched balance must not start from the gift alone), the balance moved, then a box
+ * stored. A credit is an upsert; a debit is `amount >= ?` guarded in the UPDATE itself, so
+ * nobody is overdrawn, and an account that can't cover it is SKIPPED — no box — rather than
+ * failing the rest. `paid` keeps the order of `gifts`; `skipped` is the rest. A duplicate
+ * account among `gifts` is paid once, the first entry's amount: a multi-row upsert may not
+ * name one key twice.
+ *
+ * Sends NOTHING: the caller builds the balance frame (the RESULTING total, `balance`, into
+ * the -2 bucket) and the box's announcement from the rows returned, and should push them in
+ * one hub call. Nothing here is transactional across rounds: a round that throws leaves the
+ * earlier ones paid, which the caller reports.
+ */
+export async function payTokenGifts(
+	db: D1Database,
+	gifts: readonly TokenGift[],
+	startingTokens: number
+): Promise<{ paid: PaidTokenGift[]; skipped: number[] }> {
+	const grants = startingBalances(startingTokens)
+	const paid: PaidTokenGift[] = []
+	const skipped: number[] = []
+
+	const seen = new Set<number>()
+	const unique = gifts.filter((g) => !seen.has(g.accountId) && seen.add(g.accountId))
+	for (let i = 0; i < unique.length; i += TOKEN_GIFT_CHUNK) {
+		const chunk = unique.slice(i, i + TOKEN_GIFT_CHUNK)
+		const credits = chunk.filter((g) => g.amount > 0)
+		const debits = chunk.filter((g) => g.amount < 0)
+		const holds = chunk.filter((g) => g.amount === 0)
+
+		// Seed the signup grant, then move the money: one batch, one transaction. A debit's
+		// guard is per row, so a chunk of debits is one UPDATE per distinct amount.
+		const statements: D1PreparedStatement[] = [
+			db
+				.prepare(
+					`INSERT OR IGNORE INTO balance (account_id, currency_type, amount) VALUES ${chunk
+						.flatMap(() => grants.map(() => '(?, ?, ?)'))
+						.join(', ')}`
+				)
+				.bind(
+					...chunk.flatMap((g) => grants.flatMap((b) => [g.accountId, b.currencyType, b.amount]))
+				),
+		]
+		if (credits.length > 0) {
+			statements.push(
+				db
+					.prepare(
+						`INSERT INTO balance (account_id, currency_type, amount) VALUES ${credits
+							.map(() => '(?, ?, ?)')
+							.join(', ')}
+						 ON CONFLICT (account_id, currency_type) DO UPDATE SET amount = amount + excluded.amount
+						 RETURNING account_id, amount`
+					)
+					.bind(...credits.flatMap((g) => [g.accountId, CurrencyType.RecCenterTokens, g.amount]))
+			)
+		}
+		for (const amount of new Set(debits.map((g) => g.amount))) {
+			const ids = debits.filter((g) => g.amount === amount).map((g) => g.accountId)
+			statements.push(
+				db
+					.prepare(
+						`UPDATE balance SET amount = amount - ?1
+						 WHERE currency_type = ?2 AND amount >= ?1
+						   AND account_id IN (${ids.map(() => '?').join(', ')})
+						 RETURNING account_id, amount`
+					)
+					.bind(-amount, CurrencyType.RecCenterTokens, ...ids)
+			)
+		}
+		if (holds.length > 0) {
+			statements.push(
+				db
+					.prepare(
+						`SELECT account_id, amount FROM balance
+						 WHERE currency_type = ?1 AND account_id IN (${holds.map(() => '?').join(', ')})`
+					)
+					.bind(CurrencyType.RecCenterTokens, ...holds.map((g) => g.accountId))
+			)
+		}
+		const [, ...moved] = await db.batch<{ account_id: number; amount: number }>(statements)
+		// RETURNING comes back in no promised order; key it, then walk the chunk in its order.
+		const balances = new Map(moved.flatMap((r) => r.results).map((r) => [r.account_id, r.amount]))
+		const debited = chunk.filter((g) => balances.has(g.accountId))
+		for (const g of chunk) if (!balances.has(g.accountId)) skipped.push(g.accountId)
+		if (debited.length === 0) continue
+
+		const createdAt = new Date().toISOString()
+		const { results: boxes } = await db
+			.prepare(
+				`INSERT INTO received_gift (account_id, data, created_at) VALUES ${debited
+					.map(() => '(?, ?, ?)')
+					.join(', ')} RETURNING id, account_id`
+			)
+			.bind(...debited.flatMap((g) => [g.accountId, JSON.stringify(g.content), createdAt]))
+			.all<{ id: number; account_id: number }>()
+		const giftIds = new Map(boxes.map((r) => [r.account_id, r.id]))
+
+		for (const g of debited) {
+			paid.push({ ...g, balance: balances.get(g.accountId)!, giftId: giftIds.get(g.accountId)! })
+		}
+	}
+	return { paid, skipped }
+}

apps/econ/src/discord-role-gift.ts

@@ -4,8 +4,9 @@
  * An operator maps Discord ROLE ids to a token amount — `DISCORD_ROLE_TOKENS`,
  * `<roleId>=<tokens>,<roleId>=<tokens>` — and every time the cron fires, every account
  * holding one of those roles is handed that many RecCenterTokens in a gift box. ONE box per
- * account per run: an account holding several mapped roles is paid the HIGHEST amount among
- * them, not the sum, so the map reads as tiers however Discord stacks the roles.
+ * account per run: an account holding several mapped roles — across every Discord link it
+ * has — is paid the HIGHEST amount among them, not the sum, so the map reads as tiers
+ * however Discord stacks the roles.
  *
  * HOW OFTEN is the cron schedule's business, not this module's: `triggers.crons` in
  * wrangler.jsonc says weekly, and an operator who wants a daily gift changes that line. There
@@ -42,13 +43,21 @@ import { PlatformType } from '@repo/domain/src/enums'
 
 import { getLinksForPlatform } from '../../auth/src/platform-db'
 import { NotificationType } from '../../notify/src/notification-types'
-import { ALL_PLATFORMS, creditCurrency, CurrencyType, ensureStartingBalances } from './balance-db'
+import {
+	ALL_PLATFORMS,
+	creditCurrency,
+	CurrencyType,
+	ensureStartingBalances,
+	payTokenGifts,
+	TOKEN_GIFT_CHUNK,
+} from './balance-db'
 
 import type { GiftContent } from '@repo/domain'
 import type {
 	BalanceResponsePayload,
 	GiftPackagePayload,
 } from '../../notify/src/notification-payloads'
+import type { PlayerNotification } from '../../notify/src/notifications-hub'
 import type { Env } from './context'
 
 /** The system "Coach" account — who a box the server hands over is from. */
@@ -178,19 +187,24 @@ export interface RoleGiftSummary {
 	granted: number
 	/** Tokens handed out this run, across every box. */
 	tokens: number
-	/** Grants that failed partway — logged with the account and role to sort out by hand. */
+	/** Accounts whose grant failed — logged with the account and role to sort out by hand. */
 	failed: number
 }
 
 /**
- * One run of the gift: every Discord link holding a mapped role is paid its best role's
- * tokens. Sequential: each grant is a handful of D1 writes and two hub calls, and a cron
- * invocation has a fixed subrequest budget (50 free, 1000 paid) — a community whose
- * supporters outgrow that is the point to batch it, not before.
+ * One run of the gift: every account whose Discord links hold a mapped role is paid its
+ * best role's tokens. Batched, as the staff drops are: `payTokenGifts` banks the boxes a
+ * few dozen accounts a round, and the frames go out in ONE hub call — grant by grant this
+ * was five D1 round trips and two hub calls an account, in series, against a cron
+ * invocation's fixed subrequest budget (50 free, 1,000 paid).
  *
- * `startingTokens` is the signup grant to seed before the credit — `creditCurrency` upserts
- * the balance row, so a never-touched balance would otherwise start from this gift instead
- * of the grant plus this gift.
+ * A round that throws fails every account in it: they are logged, with their roles, and
+ * counted in `failed` rather than retried — a retry that ALSO failed partway is how a
+ * balance gets credited twice — and the later rounds still run.
+ *
+ * `startingTokens` is the signup grant to seed before the credit — the credit upserts the
+ * balance row, so a never-touched balance would otherwise start from this gift instead of
+ * the grant plus this gift.
  */
 export async function grantDiscordRoleGifts(
 	env: Env,
@@ -215,19 +229,53 @@ export async function grantDiscordRoleGifts(
 	const links = await getLinksForPlatform(env.DB, PlatformType.Discord)
 	summary.links = links.length
 
+	// One box per ACCOUNT: an account with two Discord links is paid the best role across
+	// both, once. Kept in link order (oldest first) so the frames go out in a stable order.
+	const best = new Map<number, RoleTokens>()
 	for (const link of links) {
 		const role = bestRoleGift(link.roles, roles)
 		if (role === null) continue
+		const held = best.get(link.accountId)
+		if (held === undefined || role.tokens > held.tokens) best.set(link.accountId, role)
+	}
+
+	const frames: PlayerNotification[] = []
+	const due = [...best]
+	for (let i = 0; i < due.length; i += TOKEN_GIFT_CHUNK) {
+		const round = due.slice(i, i + TOKEN_GIFT_CHUNK)
 		try {
-			await grantOne(env, link.accountId, role, startingTokens)
-			summary.granted++
-			summary.tokens += role.tokens
+			const { paid } = await payTokenGifts(
+				env.DB,
+				round.map(([accountId, role]) => ({
+					accountId,
+					amount: role.tokens,
+					content: roleGiftContent(role.tokens),
+				})),
+				startingTokens
+			)
+			for (const p of paid) {
+				summary.granted++
+				summary.tokens += p.amount
+				frames.push(...giftFrames(p.accountId, p.balance, p.giftId, p.content))
+			}
 		} catch (err) {
-			// Logged with everything needed to check what did land rather than retried: a
-			// retry that ALSO failed partway is how a balance gets credited twice.
-			summary.failed++
+			summary.failed += round.length
 			console.error(
-				`discord role gift: account ${link.accountId}'s grant for role ${role.roleId} (${role.tokens} tokens) failed: ${err instanceof Error ? err.message : String(err)}`
+				`discord role gift: a round of ${round.length} grant(s) failed (${round
+					.map(
+						([accountId, role]) => `account ${accountId} role ${role.roleId} ${role.tokens} tokens`
+					)
+					.join('; ')}): ${err instanceof Error ? err.message : String(err)}`
+			)
+		}
+	}
+
+	if (frames.length > 0) {
+		try {
+			await env.RECFLARE_NOTIFICATIONS_HUB.getByName(HUB_INSTANCE).notifyPlayers(frames)
+		} catch (err) {
+			console.error(
+				`discord role gift: could not notify ${summary.granted} account(s) of their box: ${err instanceof Error ? err.message : String(err)}`
 			)
 		}
 	}
@@ -283,36 +331,59 @@ async function grantOne(
 	const content = roleGiftContent(role.tokens)
 	const gift = await createGift(env.DB, accountId, content)
 
-	const hub = env.RECFLARE_NOTIFICATIONS_HUB.getByName(HUB_INSTANCE)
 	try {
-		// The balance first, so the box's announcement lands on a total that includes it.
-		// `Balance` is the RESULTING total, into the one -2 bucket: the frame SETS it.
-		await hub.notifyPlayer(accountId, NotificationType.StorefrontBalanceUpdate, {
-			Balance: balance,
-			CurrencyType: CurrencyType.RecCenterTokens,
-			Platform: ALL_PLATFORMS,
-		} satisfies BalanceResponsePayload)
-		await hub.notifyPlayer(accountId, NotificationType.GiftPackageReceivedImmediate, {
-			Id: gift.id,
-			FromPlayerId: COACH_ACCOUNT_ID,
-			ConsumableItemDesc: content.ConsumableItemDesc,
-			AvatarItemType: content.AvatarItemType,
-			AvatarItemDesc: content.AvatarItemDesc,
-			EquipmentPrefabName: content.EquipmentPrefabName,
-			EquipmentModificationGuid: content.EquipmentModificationGuid,
-			CurrencyType: content.CurrencyType,
-			Currency: content.Currency,
-			Xp: content.Xp,
-			GiftContext: content.GiftContext ?? 0,
-			GiftRarity: content.GiftRarity,
-			Message: content.Message,
-			Platform: -1,
-			PlatformsToSpawnOn: -1,
-			BalanceType: ALL_PLATFORMS,
-		} satisfies GiftPackagePayload)
+		await env.RECFLARE_NOTIFICATIONS_HUB.getByName(HUB_INSTANCE).notifyPlayers(
+			giftFrames(accountId, balance, gift.id, content)
+		)
 	} catch (err) {
 		console.error(
 			`discord role gift: could not notify account ${accountId} of box ${gift.id}: ${err instanceof Error ? err.message : String(err)}`
 		)
 	}
 }
+
+/**
+ * The two frames announcing a stored box, in order: the balance first — the RESULTING
+ * total, into the one -2 bucket; the frame SETS it — so the box's announcement lands on a
+ * total that already includes it.
+ */
+function giftFrames(
+	accountId: number,
+	balance: number,
+	giftId: number,
+	content: GiftContent
+): PlayerNotification[] {
+	return [
+		{
+			playerId: accountId,
+			notificationType: NotificationType.StorefrontBalanceUpdate,
+			data: {
+				Balance: balance,
+				CurrencyType: CurrencyType.RecCenterTokens,
+				Platform: ALL_PLATFORMS,
+			} satisfies BalanceResponsePayload,
+		},
+		{
+			playerId: accountId,
+			notificationType: NotificationType.GiftPackageReceivedImmediate,
+			data: {
+				Id: giftId,
+				FromPlayerId: COACH_ACCOUNT_ID,
+				ConsumableItemDesc: content.ConsumableItemDesc,
+				AvatarItemType: content.AvatarItemType,
+				AvatarItemDesc: content.AvatarItemDesc,
+				EquipmentPrefabName: content.EquipmentPrefabName,
+				EquipmentModificationGuid: content.EquipmentModificationGuid,
+				CurrencyType: content.CurrencyType,
+				Currency: content.Currency,
+				Xp: content.Xp,
+				GiftContext: content.GiftContext ?? 0,
+				GiftRarity: content.GiftRarity,
+				Message: content.Message,
+				Platform: -1,
+				PlatformsToSpawnOn: -1,
+				BalanceType: ALL_PLATFORMS,
+			} satisfies GiftPackagePayload,
+		},
+	]
+}

apps/econ/src/test/integration/api.test.ts

@@ -7670,6 +7670,49 @@ describe('discord role gift', () => {
 		])
 	})
 
+	// One box per ACCOUNT, not per link: a player who linked two Discord identities is paid
+	// once, the best role across both. And more accounts than one round pays (30) are all
+	// paid once, each with their own box, with two frames apiece in link order.
+	test('pays an account with two links once, and more accounts than one round holds', async () => {
+		const many = Array.from({ length: 40 }, (_, i) => 9400 + i)
+		await onlyDiscordLinks([
+			[9341, '900000000000000341', [ROLE_A]],
+			[9341, '900000000000000342', [ROLE_B]],
+			...many.map((id): [number, string, string[]] => [id, `9000000000000${id}`, [ROLE_A]]),
+		])
+		await drainFrames()
+
+		const summary = await grantDiscordRoleGifts(giftEnv(MAP), DEFAULT_STARTING_TOKENS)
+		expect(summary).toEqual({
+			skipped: false,
+			roles: 2,
+			links: 42,
+			granted: 41,
+			tokens: 10000 + 40 * 2500,
+			failed: 0,
+		})
+		expect(await tokens(9341)).toBe(DEFAULT_STARTING_TOKENS + 10000)
+		expect(await getPendingGifts(env.DB, 9341)).toMatchObject([{ Currency: 10000 }])
+
+		const frames = await drainFrames()
+		expect(frames.map((f) => f.accountId)).toEqual([9341, 9341, ...many.flatMap((id) => [id, id])])
+		const boxIds = new Set<number>()
+		for (const id of many) {
+			expect(await tokens(id)).toBe(DEFAULT_STARTING_TOKENS + 2500)
+			const boxes = await getPendingGifts(env.DB, id)
+			expect(boxes).toHaveLength(1)
+			boxIds.add(boxes[0]!.Id)
+			const own = frames.filter((f) => f.accountId === id)
+			expect(own[0]!.payload).toEqual({
+				Balance: DEFAULT_STARTING_TOKENS + 2500,
+				CurrencyType: CurrencyType.RecCenterTokens,
+				Platform: -2,
+			})
+			expect(own[1]!.payload).toMatchObject({ Id: boxes[0]!.Id, Currency: 2500 })
+		}
+		expect(boxIds.size).toBe(many.length)
+	})
+
 	test('every run pays again — the schedule is the cadence — and a lapsed role is not paid', async () => {
 		await onlyDiscordLinks([
 			[9321, '900000000000000321', [ROLE_A]],