[api] catalog loader

CLAUDE.md

@@ -202,8 +202,10 @@ inconsistency here without checking the client first.
   FIRST-PARTY item (imported from the official export, Coach-authored, its own `OutfitType`) has
   those four NULL and is rendered from `CurrentSaves` — one built Unity assetbundle per
   `BodyType`, which the client picks by the wearer's body. The row is the record as JSON, so an
-  import is the export verbatim (`apps/api/scripts/import-custom-avatar-items.ts` writes the
+  import is the export verbatim except that `CreatorAccountId` is forced to 1, the Coach — which
-  migration; re-importing an id replaces it). Serve both through one shape: the shirt carries
+  is what files it as stock content and whom a sale pays (`just cai-load` loads it;
+  `apps/api/scripts/import-custom-avatar-items.ts` writes a migration; re-importing an id
+  replaces it). Serve both through one shape: the shirt carries
   empty `CurrentSaves`/`Tags` and null `CustomBadgeMetadata`, never a missing key. The saves
   name their assetbundles and thumbnails by bare filename; storing the record does nothing
   about serving those files.

CLI.md

@@ -76,6 +76,28 @@ This command targets the `balance` table, so it runs under `apps/econ`'s wrangle
 with `--local` that is econ's dev D1 state, which needs both `balance` and a migrated
 `account` table in it.
 
+### `cai-load` — load the first-party custom avatar items
+
+```sh
+bun runx admin cai-load                      # apps/econ/static/db/2025-1-cai.json → local
+bun runx admin cai-load --remote
+bun runx admin cai-load --file path/to/export.json --dry-run
+just cai-load --remote                       # the same, as a recipe
+```
+
+Loads an export of `CustomAvatarItem` records (a JSON array, a `{ Results }` page, or one
+record) into the `custom_avatar_item` table, each stored as the row's JSON with its
+`CreatorAccountId` forced to 1, the Coach account, whatever the export said: that is what
+files it as stock content (the storefront tab searches by creator) and what a purchase
+pays. It MERGES: an id already in the table has its row replaced, and nothing is deleted, so the
+players' own shirts, which share the table, are untouched. Re-running is safe. The load is
+verified by counting rows and probing ids from across the export; it fails loudly rather
+than report a partial load. `--dry-run` reads and validates the export and prints the
+statement count without writing.
+
+The saves in an export name assetbundles and thumbnails by bare filename; the load stores
+the names only.
+
 ### `lookup` — print an account
 
 ```sh
@@ -90,7 +112,7 @@ the account has a password, the developer role, and the moderator role.
 
 ### Selecting an account
 
-Every command except `reload-plus` targets exactly one account, by **either**:
+Every command except `reload-plus` and `cai-load` targets exactly one account, by **either**:
 
 - `--account <id>` — numeric account id
 - `--username <name>` — username (case-insensitive)

Justfile

@@ -124,3 +124,12 @@ admin *args:
 [no-cd]
 reload-plus *args:
   bun runx admin reload-plus "$@"
+
+# Load the first-party custom avatar items (apps/econ/static/db/2025-1-cai.json by default)
+# into the custom_avatar_item table. Merges: existing ids are replaced, nothing is deleted.
+# Defaults to --local; pass --remote for production, --dry-run to only validate. See CLI.md.
+[group('4. utility')]
+[positional-arguments]
+[no-cd]
+cai-load *args:
+  bun runx admin cai-load "$@"

apps/api/scripts/import-custom-avatar-items.ts

@@ -1,7 +1,7 @@
 /**
  * Turns an export of first-party custom avatar items into a D1 migration that seeds the
- * `custom_avatar_item` table with them, one `INSERT` per record, the way
+ * `custom_avatar_item` table with them, the way `apps/rooms/migrations/0002_import_rooms.sql`
- * `apps/rooms/migrations/0002_import_rooms.sql` seeds the rooms.
+ * seeds the rooms.
  *
  *   bun apps/api/scripts/import-custom-avatar-items.ts <export.json> <NNNN>
  *
@@ -9,11 +9,11 @@
  * them (a `{ Results: [...] }` page or a single record are accepted too). `NNNN` is the
  * migration number; the file is written to `apps/api/migrations/NNNN_import_custom_avatar_items.sql`.
  *
- * Each record is stored VERBATIM as the row's JSON (the table is a JSON blob per row, see
+ * For a load that needs no migration and no deploy — the usual case — use
- * src/custom-avatar-items-db.ts) minus `PurchaseInfo`, which is a store-side projection and
+ * `just admin cai-load` instead, which runs the same statements straight against a database.
- * never stored. A record whose id is already present is REPLACED, so re-running an import
+ * The statements come from src/custom-avatar-items-load.ts either way: each record stored as
- * with corrected data lands; a `DELETE` is not issued, so player-made items and previously
+ * the row's JSON minus `PurchaseInfo` and with `CreatorAccountId` forced to the Coach account
- * imported items the new export lacks are left alone.
+ * (1), an existing id REPLACED, nothing deleted.
  *
  * The saves name assetbundles and thumbnails by bare filename; the migration does nothing
  * about those files, which have to be served from wherever the client fetches them.
@@ -21,11 +21,10 @@
 
 import { readFileSync, writeFileSync } from 'node:fs'
 
-interface Record_ {
+import {
-	CustomAvatarItemId: string
+	customAvatarItemUpsertStatements,
-	PurchaseInfo?: unknown
+	readCustomAvatarItemExport,
-	[key: string]: unknown
+} from '../src/custom-avatar-items-load'
-}
 
 const [input, number] = process.argv.slice(2)
 if (!input || !number || !/^\d{4}$/.test(number)) {
@@ -33,34 +32,18 @@ if (!input || !number || !/^\d{4}$/.test(number)) {
 	process.exit(2)
 }
 
-const parsed: unknown = JSON.parse(readFileSync(input, 'utf8'))
+// Tolerate the BOM a .NET export carries (`JSON.parse` rejects U+FEFF).
-const records: Record_[] = Array.isArray(parsed)
+const records = readCustomAvatarItemExport(
-	? parsed
+	JSON.parse(readFileSync(input, 'utf8').replace(/^/, ''))
-	: parsed && typeof parsed === 'object' && Array.isArray((parsed as { Results?: unknown }).Results)
+)
-		? (parsed as { Results: Record_[] }).Results
+const lines = [
-		: [parsed as Record_]
-
-const seen = new Set<string>()
-const lines: string[] = [
 	`-- Seed first-party custom avatar items (${records.length} records) from ${input.split('/').pop()}.`,
 	'-- Generated by apps/api/scripts/import-custom-avatar-items.ts — do not edit by hand.',
-	'-- Each record is the official `CustomAvatarItem` verbatim; an existing id is replaced.',
+	'-- Each record is the official `CustomAvatarItem`, filed under the Coach account (CreatorAccountId 1);',
+	'-- an existing id is replaced.',
 	'',
+	...customAvatarItemUpsertStatements(records),
 ]
-for (const record of records) {
-	if (typeof record.CustomAvatarItemId !== 'string' || record.CustomAvatarItemId === '') {
-		throw new Error(`record without a CustomAvatarItemId: ${JSON.stringify(record).slice(0, 80)}`)
-	}
-	if (seen.has(record.CustomAvatarItemId)) {
-		throw new Error(`duplicate CustomAvatarItemId in export: ${record.CustomAvatarItemId}`)
-	}
-	seen.add(record.CustomAvatarItemId)
-	const { PurchaseInfo: _purchaseInfo, ...stored } = record
-	const json = JSON.stringify(stored).replaceAll("'", "''")
-	lines.push(
-		`INSERT INTO custom_avatar_item (data) VALUES ('${json}') ON CONFLICT(custom_avatar_item_id) DO UPDATE SET data = excluded.data;`
-	)
-}
 
 const out = decodeURIComponent(
 	new URL(`../migrations/${number}_import_custom_avatar_items.sql`, import.meta.url).pathname

apps/api/src/custom-avatar-items-db.ts

@@ -27,6 +27,8 @@
  * as JSON by 0022_custom_avatar_item_json.sql, applied under its own `migrations_table`).
  */
 
+import { COACH_ACCOUNT_ID } from './custom-avatar-items-load'
+
 /** Schema DDL (mirror of migrations/0022_custom_avatar_item_json.sql). */
 export const SCHEMA_DDL: string[] = [
 	`CREATE TABLE IF NOT EXISTS custom_avatar_item (
@@ -336,9 +338,10 @@ export async function listHotCustomAvatarItems(
 
 /**
  * The "Coach" system account — this server's stock content is authored by it, the same id the
- * `econ` worker attributes a self-buy or an anonymous gift to.
+ * `econ` worker attributes a self-buy or an anonymous gift to. Lives in the loader module so
+ * the CLI can name it without Workers types; re-exported here for the worker-side callers.
  */
-export const COACH_ACCOUNT_ID = 1
+export { COACH_ACCOUNT_ID }
 
 /** What `GET /api/customAvatarItems/v1/search` narrows the catalog by. */
 export interface CustomAvatarItemSearch {

apps/api/src/custom-avatar-items-load.ts

@@ -0,0 +1,116 @@
+/**
+ * Turning an export of custom avatar items into `custom_avatar_item` rows — the loader half of
+ * the table, used by `runx admin cai-load` (in @repo/tools) and by the migration generator
+ * (scripts/import-custom-avatar-items.ts) rather than by the worker.
+ *
+ * Separate from `custom-avatar-items-db.ts` for the reason `catalog-load.ts` is separate from
+ * `catalog-db.ts` in econ: that module types its queries with `D1Database`, a Workers type,
+ * and the loader runs in a plain Node CLI that has no such types. Everything here is pure data
+ * mapping with no imports, so both sides can use it.
+ *
+ * A row is the record as JSON (see 0022_custom_avatar_item_json.sql), so a load is the export
+ * verbatim: no per-field mapping, and a field the source adds later rides along. Two things
+ * change on the way in: `PurchaseInfo`, a store-side projection the worker fills at read time,
+ * is stripped, and `CreatorAccountId` is FORCED to the Coach account — see
+ * {@link customAvatarItemRowLiteral}.
+ */
+
+/**
+ * The "Coach" system account — this server's stock content is authored by it, the same id the
+ * `econ` worker attributes a self-buy or an anonymous gift to. Defined here rather than in
+ * `custom-avatar-items-db.ts` (which re-exports it) so the loader can name it without pulling
+ * in Workers types.
+ */
+export const COACH_ACCOUNT_ID = 1
+
+/** The least an export record must carry to be a row. Everything else is kept as-is. */
+export interface CustomAvatarItemExportRecord {
+	CustomAvatarItemId: string
+	PurchaseInfo?: unknown
+	[key: string]: unknown
+}
+
+/**
+ * The records in an export, whether it is a bare JSON array, a `{ Results: [...] }` page, or a
+ * single record. Refuses a record without an id and a repeated id — a repeat in one upsert
+ * statement is an error at the database, and a repeat across statements would silently keep
+ * whichever came last, which is a defect in the export rather than a choice to make for it.
+ */
+export function readCustomAvatarItemExport(parsed: unknown): CustomAvatarItemExportRecord[] {
+	const records: unknown[] = Array.isArray(parsed)
+		? parsed
+		: parsed &&
+			  typeof parsed === 'object' &&
+			  Array.isArray((parsed as { Results?: unknown }).Results)
+			? (parsed as { Results: unknown[] }).Results
+			: [parsed]
+	const seen = new Set<string>()
+	return records.map((record, i) => {
+		if (!record || typeof record !== 'object' || Array.isArray(record)) {
+			throw new Error(`record ${i} is not an object`)
+		}
+		const id = (record as { CustomAvatarItemId?: unknown }).CustomAvatarItemId
+		if (typeof id !== 'string' || id === '') {
+			throw new Error(`record ${i} has no CustomAvatarItemId`)
+		}
+		if (seen.has(id)) throw new Error(`duplicate CustomAvatarItemId in export: ${id}`)
+		seen.add(id)
+		return record as CustomAvatarItemExportRecord
+	})
+}
+
+/**
+ * One record as the SQL string literal its row holds: the JSON, minus `PurchaseInfo`, with
+ * `CreatorAccountId` overridden to {@link COACH_ACCOUNT_ID} whatever the export said. An
+ * imported item is first-party content by definition, and the creator id is what makes it so
+ * here: the storefront tab finds stock items by `creator_account_id = 1`
+ * (`includeCoachItems=True`), and a sale pays the creator. An export from another server
+ * carries THAT server's creator ids — an item filed under one of those would vanish from the
+ * storefront into the user-generated tab, and its price would be paid to whichever local
+ * account happened to share the number.
+ */
+export function customAvatarItemRowLiteral(record: CustomAvatarItemExportRecord): string {
+	const { PurchaseInfo: _purchaseInfo, ...stored } = record
+	const row = { ...stored, CreatorAccountId: COACH_ACCOUNT_ID }
+	return `'${JSON.stringify(row).replaceAll("'", "''")}'`
+}
+
+/**
+ * The largest statement a load builds, in bytes of SQL. D1 caps a single statement at 100 KB,
+ * and a record with three built saves runs to a few KB, so rows are grouped by size rather
+ * than by count — a fixed count would fit the small records and overflow on the large ones.
+ */
+export const CUSTOM_AVATAR_ITEM_LOAD_STATEMENT_BYTES = 64 * 1024
+
+/**
+ * The statements that upsert `records` into `custom_avatar_item`, each a multi-row INSERT
+ * within {@link CUSTOM_AVATAR_ITEM_LOAD_STATEMENT_BYTES}. An id already in the table has its
+ * row REPLACED (`ON CONFLICT ... DO UPDATE`), so a re-run with a corrected export lands and a
+ * re-run with the same one changes nothing. Nothing is deleted: rows the export does not
+ * mention — the players' own shirts among them — are left alone.
+ */
+export function customAvatarItemUpsertStatements(
+	records: CustomAvatarItemExportRecord[],
+	maxBytes = CUSTOM_AVATAR_ITEM_LOAD_STATEMENT_BYTES
+): string[] {
+	const head = 'INSERT INTO custom_avatar_item (data) VALUES\n'
+	const tail = '\nON CONFLICT(custom_avatar_item_id) DO UPDATE SET data = excluded.data;'
+	const statements: string[] = []
+	let rows: string[] = []
+	let bytes = 0
+	const flush = (): void => {
+		if (rows.length === 0) return
+		statements.push(head + rows.map((r) => `\t(${r})`).join(',\n') + tail)
+		rows = []
+		bytes = 0
+	}
+	for (const record of records) {
+		const literal = customAvatarItemRowLiteral(record)
+		// A single oversized record still gets its own statement rather than being dropped.
+		if (rows.length > 0 && bytes + literal.length > maxBytes) flush()
+		rows.push(literal)
+		bytes += literal.length + 4
+	}
+	flush()
+	return statements
+}

packages/tools/src/cmd/admin.cmd.ts

@@ -2,13 +2,18 @@ import * as readline from 'node:readline'
 import { Command } from '@commander-js/extra-typings'
 import Table from 'cli-table3'
 
+import {
+	customAvatarItemUpsertStatements,
+	readCustomAvatarItemExport,
+} from '../../../../apps/api/src/custom-avatar-items-load'
 import {
 	DEFAULT_STARTING_TOKENS,
 	PLUS_MEMBERS_SQL,
 	plusReloadSql,
 } from '../../../../apps/econ/src/currency'
-import { execSql, readRootEnv, resolveRemote, sqlStr, target } from '../d1'
+import { execSql, execSqlFile, readRootEnv, resolveRemote, sqlStr, target } from '../d1'
 import { hashPassword } from '../password'
+import { getRepoRoot } from '../path'
 
 import type { D1ExecResult } from '../d1'
 
@@ -25,6 +30,7 @@ import type { D1ExecResult } from '../d1'
  *   runx admin grant-developer --account 1 [--revoke] [--remote]
  *   runx admin grant-plus      --username alice [--revoke] [--remote]
  *   runx admin reload-plus     <amount> [--dry-run] [--remote]
+ *   runx admin cai-load        [--file <export.json>] [--dry-run] [--remote]
  */
 
 /**
@@ -256,6 +262,109 @@ const reloadPlus = new Command('reload-plus')
 		console.log(chalk.green(`✓ ${amount} tokens added to ${credited.length} account(s)`))
 	})
 
+/** The export `cai-load` reads when no `--file` is given. */
+const CUSTOM_AVATAR_ITEM_EXPORT = 'apps/econ/static/db/2025-1-cai.json'
+
+/**
+ * Load the first-party custom avatar items into `custom_avatar_item` (owned by the `api`
+ * worker). Like `runx catalog load`: the table's STRUCTURE is a migration, its first-party
+ * CONTENTS are not, and this is the reload. Each record lands as the row's JSON with its
+ * `CreatorAccountId` forced to the Coach account (1) — what makes it stock content here — (the
+ * statements come from api's `custom-avatar-items-load.ts`, the same ones the migration
+ * generator writes), an id already present is REPLACED, and nothing is deleted — the players'
+ * own shirts share the table and an export never mentions them. Re-running is always safe.
+ */
+const caiLoad = new Command('cai-load')
+	.description('Load first-party custom avatar items from an export JSON (merges; never deletes)')
+	.option(
+		'--file <path>',
+		'The export to load, relative to the repo root',
+		CUSTOM_AVATAR_ITEM_EXPORT
+	)
+	.option(
+		'--dry-run',
+		'Read and validate the export, print what would be written, change nothing',
+		false
+	)
+	.option('--local', 'Target the local dev database (the default).', false)
+	.option('--remote', 'Target the deployed database instead of the local dev database.', false)
+	.action(async (opts) => {
+		const remote = resolveRemote(opts)
+		const full = path.isAbsolute(opts.file) ? opts.file : path.join(getRepoRoot(), opts.file)
+		if (!(await fs.pathExists(full))) throw new Error(`no export at ${opts.file}`)
+		// Tolerate the BOM a .NET export carries (`JSON.parse` rejects U+FEFF).
+		const text = (await fs.readFile(full, 'utf8')).replace(/^\uFEFF/, '')
+		const records = readCustomAvatarItemExport(JSON.parse(text) as unknown)
+		if (records.length === 0)
+			throw new Error('the export holds no records — refusing to call that a load')
+		const statements = customAvatarItemUpsertStatements(records)
+		const sql = statements.join('\n')
+
+		console.log(
+			`Merging ${records.length} custom avatar items from ${opts.file} into ${target(remote)}`
+		)
+		if (opts.dryRun) {
+			console.log(
+				chalk.cyan(
+					`--dry-run: built ${statements.length} statements (${(sql.length / 1024).toFixed(0)} KB); nothing was written.`
+				)
+			)
+			return
+		}
+
+		const count = async (): Promise<number> =>
+			Number(
+				(await execSql('SELECT COUNT(*) AS n FROM custom_avatar_item', remote, 'api')).results[0]
+					?.n ?? 0
+			)
+		const before = await count()
+
+		// Written to a temp file rather than passed as `--command`: a multi-megabyte argv is not
+		// something to rely on, and `--file` is the path wrangler batches (or, remotely, imports).
+		const file = path.join(os.tmpdir(), `recflare-cai-${Date.now()}.sql`)
+		await fs.writeFile(file, sql)
+		try {
+			await execSqlFile(file, remote, 'api')
+		} finally {
+			await fs.remove(file)
+		}
+
+		// Prove the load landed rather than trusting the exit: wrangler's meta carries no usable
+		// row count, and a file that fails partway leaves a partial load rather than an error.
+		// A merge only adds, so the table must hold at least what was written; then spot-check
+		// ids from across the export, the last one mattering most.
+		const after = await count()
+		if (after < records.length) {
+			throw new Error(
+				`load did not land: expected at least ${records.length} rows, the table holds ${after}. Re-run it.`
+			)
+		}
+		const probes = [
+			records[0],
+			records[Math.floor(records.length / 2)],
+			records[records.length - 1],
+		].map((r) => `'${sqlStr(r.CustomAvatarItemId)}'`)
+		const found = Number(
+			(
+				await execSql(
+					`SELECT COUNT(*) AS n FROM custom_avatar_item WHERE custom_avatar_item_id IN (${probes.join(', ')})`,
+					remote,
+					'api'
+				)
+			).results[0]?.n ?? 0
+		)
+		if (found !== probes.length) {
+			throw new Error(
+				`load did not land: ${probes.length - found} of ${probes.length} probe rows are missing. Re-run it.`
+			)
+		}
+		const added = after - before
+		console.log(
+			`${added} new, ${records.length - added} replaced in place, 0 removed (${after} rows now)`
+		)
+		console.log(chalk.green(`✓ custom avatar items loaded into ${target(remote)}`))
+	})
+
 const lookup = new Command('lookup')
 	.description('Print an account by id or username')
 	.option('--account <id>', 'Account id to look up')
@@ -308,6 +417,7 @@ export const adminCmd = new Command('admin')
 	.addCommand(grantModerator)
 	.addCommand(grantPlus)
 	.addCommand(reloadPlus)
+	.addCommand(caiLoad)
 	.addCommand(lookup)
 	.addHelpText(
 		'after',
@@ -325,5 +435,7 @@ Examples:
   $ runx admin grant-plus --username alice          # Rec Room Plus; takes effect next login
   $ runx admin reload-plus 1000 --remote            # +1000 tokens to every Plus subscriber
   $ runx admin reload-plus 1000 --dry-run           # just list them
+  $ runx admin cai-load                             # merge the first-party custom items (local)
+  $ runx admin cai-load --remote --file path/to/export.json
   $ runx admin lookup --username alice --remote`
 	)