From 61bf09d18e1a7fa1f8964736a48434a77dd3078b Mon Sep 17 00:00:00 2001 From: Devin Zuczek Date: Tue, 15 Sep 2026 23:39:38 -0400 Subject: [PATCH] [api] catalog loader --- CLAUDE.md | 6 +- CLI.md | 24 +++- Justfile | 9 ++ .../api/scripts/import-custom-avatar-items.ts | 55 +++------ apps/api/src/custom-avatar-items-db.ts | 7 +- apps/api/src/custom-avatar-items-load.ts | 116 ++++++++++++++++++ packages/tools/src/cmd/admin.cmd.ts | 114 ++++++++++++++++- 7 files changed, 289 insertions(+), 42 deletions(-) create mode 100644 apps/api/src/custom-avatar-items-load.ts diff --git a/CLAUDE.md b/CLAUDE.md index 5db10c8..fd06eb3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -202,8 +202,10 @@ inconsistency here without checking the client first. FIRST-PARTY item (imported from the official export, Coach-authored, its own `OutfitType`) has those four NULL and is rendered from `CurrentSaves` — one built Unity assetbundle per `BodyType`, which the client picks by the wearer's body. The row is the record as JSON, so an - import is the export verbatim (`apps/api/scripts/import-custom-avatar-items.ts` writes the - migration; re-importing an id replaces it). Serve both through one shape: the shirt carries + import is the export verbatim except that `CreatorAccountId` is forced to 1, the Coach — which + is what files it as stock content and whom a sale pays (`just cai-load` loads it; + `apps/api/scripts/import-custom-avatar-items.ts` writes a migration; re-importing an id + replaces it). Serve both through one shape: the shirt carries empty `CurrentSaves`/`Tags` and null `CustomBadgeMetadata`, never a missing key. The saves name their assetbundles and thumbnails by bare filename; storing the record does nothing about serving those files. diff --git a/CLI.md b/CLI.md index 472204f..a8f9bc7 100644 --- a/CLI.md +++ b/CLI.md @@ -76,6 +76,28 @@ This command targets the `balance` table, so it runs under `apps/econ`'s wrangle with `--local` that is econ's dev D1 state, which needs both `balance` and a migrated `account` table in it. +### `cai-load` — load the first-party custom avatar items + +```sh +bun runx admin cai-load # apps/econ/static/db/2025-1-cai.json → local +bun runx admin cai-load --remote +bun runx admin cai-load --file path/to/export.json --dry-run +just cai-load --remote # the same, as a recipe +``` + +Loads an export of `CustomAvatarItem` records (a JSON array, a `{ Results }` page, or one +record) into the `custom_avatar_item` table, each stored as the row's JSON with its +`CreatorAccountId` forced to 1, the Coach account, whatever the export said: that is what +files it as stock content (the storefront tab searches by creator) and what a purchase +pays. It MERGES: an id already in the table has its row replaced, and nothing is deleted, so the +players' own shirts, which share the table, are untouched. Re-running is safe. The load is +verified by counting rows and probing ids from across the export; it fails loudly rather +than report a partial load. `--dry-run` reads and validates the export and prints the +statement count without writing. + +The saves in an export name assetbundles and thumbnails by bare filename; the load stores +the names only. + ### `lookup` — print an account ```sh @@ -90,7 +112,7 @@ the account has a password, the developer role, and the moderator role. ### Selecting an account -Every command except `reload-plus` targets exactly one account, by **either**: +Every command except `reload-plus` and `cai-load` targets exactly one account, by **either**: - `--account ` — numeric account id - `--username ` — username (case-insensitive) diff --git a/Justfile b/Justfile index a7c84b8..85f3eca 100644 --- a/Justfile +++ b/Justfile @@ -124,3 +124,12 @@ admin *args: [no-cd] reload-plus *args: bun runx admin reload-plus "$@" + +# Load the first-party custom avatar items (apps/econ/static/db/2025-1-cai.json by default) +# into the custom_avatar_item table. Merges: existing ids are replaced, nothing is deleted. +# Defaults to --local; pass --remote for production, --dry-run to only validate. See CLI.md. +[group('4. utility')] +[positional-arguments] +[no-cd] +cai-load *args: + bun runx admin cai-load "$@" diff --git a/apps/api/scripts/import-custom-avatar-items.ts b/apps/api/scripts/import-custom-avatar-items.ts index c83e2f9..ede6f1a 100644 --- a/apps/api/scripts/import-custom-avatar-items.ts +++ b/apps/api/scripts/import-custom-avatar-items.ts @@ -1,7 +1,7 @@ /** * Turns an export of first-party custom avatar items into a D1 migration that seeds the - * `custom_avatar_item` table with them, one `INSERT` per record, the way - * `apps/rooms/migrations/0002_import_rooms.sql` seeds the rooms. + * `custom_avatar_item` table with them, the way `apps/rooms/migrations/0002_import_rooms.sql` + * seeds the rooms. * * bun apps/api/scripts/import-custom-avatar-items.ts * @@ -9,11 +9,11 @@ * them (a `{ Results: [...] }` page or a single record are accepted too). `NNNN` is the * migration number; the file is written to `apps/api/migrations/NNNN_import_custom_avatar_items.sql`. * - * Each record is stored VERBATIM as the row's JSON (the table is a JSON blob per row, see - * src/custom-avatar-items-db.ts) minus `PurchaseInfo`, which is a store-side projection and - * never stored. A record whose id is already present is REPLACED, so re-running an import - * with corrected data lands; a `DELETE` is not issued, so player-made items and previously - * imported items the new export lacks are left alone. + * For a load that needs no migration and no deploy — the usual case — use + * `just admin cai-load` instead, which runs the same statements straight against a database. + * The statements come from src/custom-avatar-items-load.ts either way: each record stored as + * the row's JSON minus `PurchaseInfo` and with `CreatorAccountId` forced to the Coach account + * (1), an existing id REPLACED, nothing deleted. * * The saves name assetbundles and thumbnails by bare filename; the migration does nothing * about those files, which have to be served from wherever the client fetches them. @@ -21,11 +21,10 @@ import { readFileSync, writeFileSync } from 'node:fs' -interface Record_ { - CustomAvatarItemId: string - PurchaseInfo?: unknown - [key: string]: unknown -} +import { + customAvatarItemUpsertStatements, + readCustomAvatarItemExport, +} from '../src/custom-avatar-items-load' const [input, number] = process.argv.slice(2) if (!input || !number || !/^\d{4}$/.test(number)) { @@ -33,34 +32,18 @@ if (!input || !number || !/^\d{4}$/.test(number)) { process.exit(2) } -const parsed: unknown = JSON.parse(readFileSync(input, 'utf8')) -const records: Record_[] = Array.isArray(parsed) - ? parsed - : parsed && typeof parsed === 'object' && Array.isArray((parsed as { Results?: unknown }).Results) - ? (parsed as { Results: Record_[] }).Results - : [parsed as Record_] - -const seen = new Set() -const lines: string[] = [ +// Tolerate the BOM a .NET export carries (`JSON.parse` rejects U+FEFF). +const records = readCustomAvatarItemExport( + JSON.parse(readFileSync(input, 'utf8').replace(/^/, '')) +) +const lines = [ `-- Seed first-party custom avatar items (${records.length} records) from ${input.split('/').pop()}.`, '-- Generated by apps/api/scripts/import-custom-avatar-items.ts — do not edit by hand.', - '-- Each record is the official `CustomAvatarItem` verbatim; an existing id is replaced.', + '-- Each record is the official `CustomAvatarItem`, filed under the Coach account (CreatorAccountId 1);', + '-- an existing id is replaced.', '', + ...customAvatarItemUpsertStatements(records), ] -for (const record of records) { - if (typeof record.CustomAvatarItemId !== 'string' || record.CustomAvatarItemId === '') { - throw new Error(`record without a CustomAvatarItemId: ${JSON.stringify(record).slice(0, 80)}`) - } - if (seen.has(record.CustomAvatarItemId)) { - throw new Error(`duplicate CustomAvatarItemId in export: ${record.CustomAvatarItemId}`) - } - seen.add(record.CustomAvatarItemId) - const { PurchaseInfo: _purchaseInfo, ...stored } = record - const json = JSON.stringify(stored).replaceAll("'", "''") - lines.push( - `INSERT INTO custom_avatar_item (data) VALUES ('${json}') ON CONFLICT(custom_avatar_item_id) DO UPDATE SET data = excluded.data;` - ) -} const out = decodeURIComponent( new URL(`../migrations/${number}_import_custom_avatar_items.sql`, import.meta.url).pathname diff --git a/apps/api/src/custom-avatar-items-db.ts b/apps/api/src/custom-avatar-items-db.ts index bb33c9e..eaa848b 100644 --- a/apps/api/src/custom-avatar-items-db.ts +++ b/apps/api/src/custom-avatar-items-db.ts @@ -27,6 +27,8 @@ * as JSON by 0022_custom_avatar_item_json.sql, applied under its own `migrations_table`). */ +import { COACH_ACCOUNT_ID } from './custom-avatar-items-load' + /** Schema DDL (mirror of migrations/0022_custom_avatar_item_json.sql). */ export const SCHEMA_DDL: string[] = [ `CREATE TABLE IF NOT EXISTS custom_avatar_item ( @@ -336,9 +338,10 @@ export async function listHotCustomAvatarItems( /** * The "Coach" system account — this server's stock content is authored by it, the same id the - * `econ` worker attributes a self-buy or an anonymous gift to. + * `econ` worker attributes a self-buy or an anonymous gift to. Lives in the loader module so + * the CLI can name it without Workers types; re-exported here for the worker-side callers. */ -export const COACH_ACCOUNT_ID = 1 +export { COACH_ACCOUNT_ID } /** What `GET /api/customAvatarItems/v1/search` narrows the catalog by. */ export interface CustomAvatarItemSearch { diff --git a/apps/api/src/custom-avatar-items-load.ts b/apps/api/src/custom-avatar-items-load.ts new file mode 100644 index 0000000..e50d966 --- /dev/null +++ b/apps/api/src/custom-avatar-items-load.ts @@ -0,0 +1,116 @@ +/** + * Turning an export of custom avatar items into `custom_avatar_item` rows — the loader half of + * the table, used by `runx admin cai-load` (in @repo/tools) and by the migration generator + * (scripts/import-custom-avatar-items.ts) rather than by the worker. + * + * Separate from `custom-avatar-items-db.ts` for the reason `catalog-load.ts` is separate from + * `catalog-db.ts` in econ: that module types its queries with `D1Database`, a Workers type, + * and the loader runs in a plain Node CLI that has no such types. Everything here is pure data + * mapping with no imports, so both sides can use it. + * + * A row is the record as JSON (see 0022_custom_avatar_item_json.sql), so a load is the export + * verbatim: no per-field mapping, and a field the source adds later rides along. Two things + * change on the way in: `PurchaseInfo`, a store-side projection the worker fills at read time, + * is stripped, and `CreatorAccountId` is FORCED to the Coach account — see + * {@link customAvatarItemRowLiteral}. + */ + +/** + * The "Coach" system account — this server's stock content is authored by it, the same id the + * `econ` worker attributes a self-buy or an anonymous gift to. Defined here rather than in + * `custom-avatar-items-db.ts` (which re-exports it) so the loader can name it without pulling + * in Workers types. + */ +export const COACH_ACCOUNT_ID = 1 + +/** The least an export record must carry to be a row. Everything else is kept as-is. */ +export interface CustomAvatarItemExportRecord { + CustomAvatarItemId: string + PurchaseInfo?: unknown + [key: string]: unknown +} + +/** + * The records in an export, whether it is a bare JSON array, a `{ Results: [...] }` page, or a + * single record. Refuses a record without an id and a repeated id — a repeat in one upsert + * statement is an error at the database, and a repeat across statements would silently keep + * whichever came last, which is a defect in the export rather than a choice to make for it. + */ +export function readCustomAvatarItemExport(parsed: unknown): CustomAvatarItemExportRecord[] { + const records: unknown[] = Array.isArray(parsed) + ? parsed + : parsed && + typeof parsed === 'object' && + Array.isArray((parsed as { Results?: unknown }).Results) + ? (parsed as { Results: unknown[] }).Results + : [parsed] + const seen = new Set() + return records.map((record, i) => { + if (!record || typeof record !== 'object' || Array.isArray(record)) { + throw new Error(`record ${i} is not an object`) + } + const id = (record as { CustomAvatarItemId?: unknown }).CustomAvatarItemId + if (typeof id !== 'string' || id === '') { + throw new Error(`record ${i} has no CustomAvatarItemId`) + } + if (seen.has(id)) throw new Error(`duplicate CustomAvatarItemId in export: ${id}`) + seen.add(id) + return record as CustomAvatarItemExportRecord + }) +} + +/** + * One record as the SQL string literal its row holds: the JSON, minus `PurchaseInfo`, with + * `CreatorAccountId` overridden to {@link COACH_ACCOUNT_ID} whatever the export said. An + * imported item is first-party content by definition, and the creator id is what makes it so + * here: the storefront tab finds stock items by `creator_account_id = 1` + * (`includeCoachItems=True`), and a sale pays the creator. An export from another server + * carries THAT server's creator ids — an item filed under one of those would vanish from the + * storefront into the user-generated tab, and its price would be paid to whichever local + * account happened to share the number. + */ +export function customAvatarItemRowLiteral(record: CustomAvatarItemExportRecord): string { + const { PurchaseInfo: _purchaseInfo, ...stored } = record + const row = { ...stored, CreatorAccountId: COACH_ACCOUNT_ID } + return `'${JSON.stringify(row).replaceAll("'", "''")}'` +} + +/** + * The largest statement a load builds, in bytes of SQL. D1 caps a single statement at 100 KB, + * and a record with three built saves runs to a few KB, so rows are grouped by size rather + * than by count — a fixed count would fit the small records and overflow on the large ones. + */ +export const CUSTOM_AVATAR_ITEM_LOAD_STATEMENT_BYTES = 64 * 1024 + +/** + * The statements that upsert `records` into `custom_avatar_item`, each a multi-row INSERT + * within {@link CUSTOM_AVATAR_ITEM_LOAD_STATEMENT_BYTES}. An id already in the table has its + * row REPLACED (`ON CONFLICT ... DO UPDATE`), so a re-run with a corrected export lands and a + * re-run with the same one changes nothing. Nothing is deleted: rows the export does not + * mention — the players' own shirts among them — are left alone. + */ +export function customAvatarItemUpsertStatements( + records: CustomAvatarItemExportRecord[], + maxBytes = CUSTOM_AVATAR_ITEM_LOAD_STATEMENT_BYTES +): string[] { + const head = 'INSERT INTO custom_avatar_item (data) VALUES\n' + const tail = '\nON CONFLICT(custom_avatar_item_id) DO UPDATE SET data = excluded.data;' + const statements: string[] = [] + let rows: string[] = [] + let bytes = 0 + const flush = (): void => { + if (rows.length === 0) return + statements.push(head + rows.map((r) => `\t(${r})`).join(',\n') + tail) + rows = [] + bytes = 0 + } + for (const record of records) { + const literal = customAvatarItemRowLiteral(record) + // A single oversized record still gets its own statement rather than being dropped. + if (rows.length > 0 && bytes + literal.length > maxBytes) flush() + rows.push(literal) + bytes += literal.length + 4 + } + flush() + return statements +} diff --git a/packages/tools/src/cmd/admin.cmd.ts b/packages/tools/src/cmd/admin.cmd.ts index 52c8dbe..2c6dd39 100644 --- a/packages/tools/src/cmd/admin.cmd.ts +++ b/packages/tools/src/cmd/admin.cmd.ts @@ -2,13 +2,18 @@ import * as readline from 'node:readline' import { Command } from '@commander-js/extra-typings' import Table from 'cli-table3' +import { + customAvatarItemUpsertStatements, + readCustomAvatarItemExport, +} from '../../../../apps/api/src/custom-avatar-items-load' import { DEFAULT_STARTING_TOKENS, PLUS_MEMBERS_SQL, plusReloadSql, } from '../../../../apps/econ/src/currency' -import { execSql, readRootEnv, resolveRemote, sqlStr, target } from '../d1' +import { execSql, execSqlFile, readRootEnv, resolveRemote, sqlStr, target } from '../d1' import { hashPassword } from '../password' +import { getRepoRoot } from '../path' import type { D1ExecResult } from '../d1' @@ -25,6 +30,7 @@ import type { D1ExecResult } from '../d1' * runx admin grant-developer --account 1 [--revoke] [--remote] * runx admin grant-plus --username alice [--revoke] [--remote] * runx admin reload-plus [--dry-run] [--remote] + * runx admin cai-load [--file ] [--dry-run] [--remote] */ /** @@ -256,6 +262,109 @@ const reloadPlus = new Command('reload-plus') console.log(chalk.green(`✓ ${amount} tokens added to ${credited.length} account(s)`)) }) +/** The export `cai-load` reads when no `--file` is given. */ +const CUSTOM_AVATAR_ITEM_EXPORT = 'apps/econ/static/db/2025-1-cai.json' + +/** + * Load the first-party custom avatar items into `custom_avatar_item` (owned by the `api` + * worker). Like `runx catalog load`: the table's STRUCTURE is a migration, its first-party + * CONTENTS are not, and this is the reload. Each record lands as the row's JSON with its + * `CreatorAccountId` forced to the Coach account (1) — what makes it stock content here — (the + * statements come from api's `custom-avatar-items-load.ts`, the same ones the migration + * generator writes), an id already present is REPLACED, and nothing is deleted — the players' + * own shirts share the table and an export never mentions them. Re-running is always safe. + */ +const caiLoad = new Command('cai-load') + .description('Load first-party custom avatar items from an export JSON (merges; never deletes)') + .option( + '--file ', + 'The export to load, relative to the repo root', + CUSTOM_AVATAR_ITEM_EXPORT + ) + .option( + '--dry-run', + 'Read and validate the export, print what would be written, change nothing', + false + ) + .option('--local', 'Target the local dev database (the default).', false) + .option('--remote', 'Target the deployed database instead of the local dev database.', false) + .action(async (opts) => { + const remote = resolveRemote(opts) + const full = path.isAbsolute(opts.file) ? opts.file : path.join(getRepoRoot(), opts.file) + if (!(await fs.pathExists(full))) throw new Error(`no export at ${opts.file}`) + // Tolerate the BOM a .NET export carries (`JSON.parse` rejects U+FEFF). + const text = (await fs.readFile(full, 'utf8')).replace(/^\uFEFF/, '') + const records = readCustomAvatarItemExport(JSON.parse(text) as unknown) + if (records.length === 0) + throw new Error('the export holds no records — refusing to call that a load') + const statements = customAvatarItemUpsertStatements(records) + const sql = statements.join('\n') + + console.log( + `Merging ${records.length} custom avatar items from ${opts.file} into ${target(remote)}` + ) + if (opts.dryRun) { + console.log( + chalk.cyan( + `--dry-run: built ${statements.length} statements (${(sql.length / 1024).toFixed(0)} KB); nothing was written.` + ) + ) + return + } + + const count = async (): Promise => + Number( + (await execSql('SELECT COUNT(*) AS n FROM custom_avatar_item', remote, 'api')).results[0] + ?.n ?? 0 + ) + const before = await count() + + // Written to a temp file rather than passed as `--command`: a multi-megabyte argv is not + // something to rely on, and `--file` is the path wrangler batches (or, remotely, imports). + const file = path.join(os.tmpdir(), `recflare-cai-${Date.now()}.sql`) + await fs.writeFile(file, sql) + try { + await execSqlFile(file, remote, 'api') + } finally { + await fs.remove(file) + } + + // Prove the load landed rather than trusting the exit: wrangler's meta carries no usable + // row count, and a file that fails partway leaves a partial load rather than an error. + // A merge only adds, so the table must hold at least what was written; then spot-check + // ids from across the export, the last one mattering most. + const after = await count() + if (after < records.length) { + throw new Error( + `load did not land: expected at least ${records.length} rows, the table holds ${after}. Re-run it.` + ) + } + const probes = [ + records[0], + records[Math.floor(records.length / 2)], + records[records.length - 1], + ].map((r) => `'${sqlStr(r.CustomAvatarItemId)}'`) + const found = Number( + ( + await execSql( + `SELECT COUNT(*) AS n FROM custom_avatar_item WHERE custom_avatar_item_id IN (${probes.join(', ')})`, + remote, + 'api' + ) + ).results[0]?.n ?? 0 + ) + if (found !== probes.length) { + throw new Error( + `load did not land: ${probes.length - found} of ${probes.length} probe rows are missing. Re-run it.` + ) + } + const added = after - before + console.log( + `${added} new, ${records.length - added} replaced in place, 0 removed (${after} rows now)` + ) + console.log(chalk.green(`✓ custom avatar items loaded into ${target(remote)}`)) + }) + const lookup = new Command('lookup') .description('Print an account by id or username') .option('--account ', 'Account id to look up') @@ -308,6 +417,7 @@ export const adminCmd = new Command('admin') .addCommand(grantModerator) .addCommand(grantPlus) .addCommand(reloadPlus) + .addCommand(caiLoad) .addCommand(lookup) .addHelpText( 'after', @@ -325,5 +435,7 @@ Examples: $ runx admin grant-plus --username alice # Rec Room Plus; takes effect next login $ runx admin reload-plus 1000 --remote # +1000 tokens to every Plus subscriber $ runx admin reload-plus 1000 --dry-run # just list them + $ runx admin cai-load # merge the first-party custom items (local) + $ runx admin cai-load --remote --file path/to/export.json $ runx admin lookup --username alice --remote` )