From f8ca01dd4dd562dc24efd76901c89623d2a46b04 Mon Sep 17 00:00:00 2001 From: Devin Zuczek Date: Thu, 27 Aug 2026 22:43:41 -0400 Subject: [PATCH] [packages] tools for catalog building --- packages/tools/src/bin/runx.cmd.ts | 4 + packages/tools/src/cmd/admin.cmd.ts | 90 +----- packages/tools/src/cmd/catalog.cmd.ts | 347 +++++++++++++++++++++++ packages/tools/src/cmd/storefront.cmd.ts | 196 +++++++++++++ packages/tools/src/d1.ts | 122 ++++++++ 5 files changed, 680 insertions(+), 79 deletions(-) create mode 100644 packages/tools/src/cmd/catalog.cmd.ts create mode 100644 packages/tools/src/cmd/storefront.cmd.ts create mode 100644 packages/tools/src/d1.ts diff --git a/packages/tools/src/bin/runx.cmd.ts b/packages/tools/src/bin/runx.cmd.ts index 6de3768..437f2a0 100644 --- a/packages/tools/src/bin/runx.cmd.ts +++ b/packages/tools/src/bin/runx.cmd.ts @@ -5,11 +5,13 @@ import { catchProcessError } from '@jahands/cli-tools/proc' import { adminCmd } from '../cmd/admin.cmd' import { buildCmd } from '../cmd/build.cmd' +import { catalogCmd } from '../cmd/catalog.cmd' import { checkCmd } from '../cmd/check.cmd' import { ciCmd } from '../cmd/ci.cmd' import { devCmd } from '../cmd/dev.cmd' import { fixCmd } from '../cmd/fix.cmd' import { shfmtCmd } from '../cmd/shfmt.cmd' +import { storefrontCmd } from '../cmd/storefront.cmd' import { updateCmd } from '../cmd/update.cmd' program @@ -20,6 +22,8 @@ program // a typescript CLI is nicer for more complex things. .addCommand(adminCmd) + .addCommand(catalogCmd) + .addCommand(storefrontCmd) .addCommand(fixCmd) .addCommand(buildCmd) .addCommand(checkCmd) diff --git a/packages/tools/src/cmd/admin.cmd.ts b/packages/tools/src/cmd/admin.cmd.ts index f47b259..75ca3cb 100644 --- a/packages/tools/src/cmd/admin.cmd.ts +++ b/packages/tools/src/cmd/admin.cmd.ts @@ -1,11 +1,12 @@ import * as readline from 'node:readline' - import { Command } from '@commander-js/extra-typings' import Table from 'cli-table3' -import { getRepoRoot } from '../path' +import { execSql, resolveRemote, sqlStr, target } from '../d1' import { hashPassword } from '../password' +import type { D1ExecResult } from '../d1' + /** * Operator-facing admin tools for the shared `recflare` D1 database. Each command * shells out to `wrangler d1 execute recflare` — no running worker or auth token @@ -19,18 +20,6 @@ import { hashPassword } from '../password' * runx admin grant-developer --account 1 [--revoke] [--remote] */ -/** The one shared database every D1-backed worker binds. */ -const DB_NAME = 'recflare' - -interface D1ExecResult { - results: Array> - success: boolean - meta: { changes?: number; rows_read?: number } -} - -/** Escape a value for embedding inside a single-quoted SQL string literal. */ -const sqlStr = (s: string): string => s.replace(/'/g, "''") - /** * Resolve the account selector into a SQL WHERE fragment. Exactly one of * `--account` / `--username` must be given. Account ids are validated numeric; @@ -50,59 +39,6 @@ function whereClause(account?: string, username?: string): { where: string; labe } } -/** The deployed D1's real id, from the environment or the gitignored root .env. */ -async function getRemoteD1Id(): Promise { - if (process.env.RECFLARE_D1) return process.env.RECFLARE_D1 - const envPath = path.join(getRepoRoot(), '.env') - if (await fs.pathExists(envPath)) { - const content = await fs.readFile(envPath, 'utf8') - const m = content.match(/^\s*RECFLARE_D1\s*=\s*(.+?)\s*$/m) - if (m) return m[1].replace(/^["']|["']$/g, '') - } - throw new Error('RECFLARE_D1 is not set — add the recflare D1 id to .env (see .env.example)') -} - -/** - * Run a SQL statement against the shared database via wrangler. Runs from the auth - * worker's directory (it owns the accounts schema and binds the DB). For `--remote` - * the committed wrangler.jsonc's "local" database_id placeholder is spliced with the - * real id into a gitignored generated config — exactly like run-wrangler-migrate. - */ -async function execSql(sql: string, remote: boolean): Promise { - const authDir = path.join(getRepoRoot(), 'apps', 'auth') - cd(authDir) - - const args = ['d1', 'execute', DB_NAME, '--command', sql, '--json'] - let cleanup: (() => Promise) | undefined - - if (remote) { - const id = await getRemoteD1Id() - const src = await fs.readFile(path.join(authDir, 'wrangler.jsonc'), 'utf8') - const generated = src.replace(/("database_id"\s*:\s*")[^"]*(")/, `$1${id}$2`) - const genPath = path.join(authDir, 'wrangler.generated.jsonc') - await fs.writeFile(genPath, generated) - cleanup = () => fs.remove(genPath) - args.push('--config', 'wrangler.generated.jsonc', '--remote') - } else { - args.push('--local') - } - - try { - // Via `pnpm exec` so wrangler resolves from the auth worker's node_modules - // (it isn't a dependency of @repo/tools, so it's not on this process's PATH). - const out = await $`pnpm exec wrangler ${args}`.quiet() - // wrangler --json prints a one-element array of results to stdout. - const start = out.stdout.indexOf('[') - if (start === -1) throw new Error(`unexpected d1 execute output:\n${out.stdout}`) - const parsed = JSON.parse(out.stdout.slice(start)) as D1ExecResult[] - const first = parsed[0] - if (!first) throw new Error(`empty d1 execute result:\n${out.stdout}`) - return first - } finally { - if (cleanup) await cleanup() - } -} - /** Prompt for a line of input without echoing what's typed (for passwords). */ function promptHidden(query: string): Promise { return new Promise((resolve) => { @@ -131,7 +67,9 @@ async function resolvePassword(flag?: string): Promise { if (!process.stdin.isTTY) { const chunks: Buffer[] = [] for await (const chunk of process.stdin) chunks.push(chunk as Buffer) - const piped = Buffer.concat(chunks).toString('utf8').replace(/\r?\n$/, '') + const piped = Buffer.concat(chunks) + .toString('utf8') + .replace(/\r?\n$/, '') if (piped === '') throw new Error('no password provided on stdin') return piped } @@ -142,16 +80,6 @@ async function resolvePassword(flag?: string): Promise { return first } -/** A short, loud label for which database a command is about to touch. */ -const target = (remote: boolean): string => - remote ? chalk.red(`${DB_NAME} (remote)`) : chalk.cyan(`${DB_NAME} (local)`) - -/** Resolve the --local/--remote target flags. Local is the default. */ -function resolveRemote(opts: { local?: boolean; remote?: boolean }): boolean { - if (opts.local && opts.remote) throw new Error('pass at most one of --local / --remote') - return opts.remote === true -} - /** * Fail when a WHERE-scoped UPDATE matched no row (i.e. no such account). Relies on * the statement's `RETURNING account_id` — wrangler's `--json` meta doesn't reliably @@ -251,7 +179,11 @@ const lookup = new Command('lookup') return } const asText = (v: unknown): string => - v == null ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v as number | string | boolean) + v == null + ? '' + : typeof v === 'object' + ? JSON.stringify(v) + : String(v as number | string | boolean) const boolKeys = new Set(['hasPassword', 'isDeveloper', 'isModerator']) const table = new Table() for (const [key, value] of Object.entries(row)) { diff --git a/packages/tools/src/cmd/catalog.cmd.ts b/packages/tools/src/cmd/catalog.cmd.ts new file mode 100644 index 0000000..883eb72 --- /dev/null +++ b/packages/tools/src/cmd/catalog.cmd.ts @@ -0,0 +1,347 @@ +import { Command } from '@commander-js/extra-typings' +import Table from 'cli-table3' + +import { + buildCatalogLoad, + CATALOG_ID_BASE, + CATALOG_INSERT_COLUMNS, +} from '../../../../apps/econ/src/catalog-load' +import { execSql, execSqlFile, resolveRemote, target } from '../d1' +import { getRepoRoot } from '../path' + +import type { + AvatarItemCapture, + CatalogCollision, + CatalogLoadRow, + CatalogValue, + SkinCapture, +} from '../../../../apps/econ/src/catalog-load' + +/** + * Load the item catalog into the shared `recflare` D1 database. + * + * The `catalog` table's STRUCTURE is a migration (apps/econ/migrations/0015_catalog.sql); its + * CONTENTS are not. The game's item list changes without the schema changing, so shipping the + * rows as a migration would mean a migration and a deploy per refresh, an ever-growing pile of + * near-identical data migrations, and no way to reload without writing another one. This + * command is the reload. + * + * runx catalog load [--remote] [--replace] [--dry-run] + * runx catalog check + * + * The mapping from capture to row lives in the econ worker's `catalog-load.ts`, beside the + * schema it fills, so a column added there and a column added to the loader cannot drift. That + * module deliberately touches no Workers types, which is what lets this Node CLI import it. + */ + +/** The worker that owns the table — whose wrangler.jsonc and node_modules wrangler uses. */ +const OWNER = 'econ' + +const AVATAR_ITEMS = 'apps/econ/static/db/avatar-items.json' +const SKINS = 'apps/econ/static/db/skins.json' + +/** + * Rows per INSERT. Batched because thousands of single-row statements parse far more slowly + * than a few dozen multi-row ones, and D1 charges per statement on the remote path. + */ +const CHUNK = 100 + +/** + * One SQL literal. Booleans become 1/0 (SQLite has no boolean), `undefined` is treated as the + * NULL it stands for — a key the capture omitted rather than set — and quotes are doubled. + * Item names are the only free text here, but escaping is what makes running this against a + * fresh capture safe rather than lucky. + */ +function lit(v: CatalogValue): string { + if (v === null || v === undefined) return 'NULL' + if (typeof v === 'boolean') return v ? '1' : '0' + if (typeof v === 'number') return String(v) + return `'${v.replaceAll("'", "''")}'` +} + +/** Read one capture, tolerating the BOM the exports carry (`JSON.parse` rejects U+FEFF). */ +async function readCapture(relPath: string): Promise { + const full = path.join(getRepoRoot(), relPath) + if (!(await fs.pathExists(full))) throw new Error(`no capture at ${relPath}`) + const text = (await fs.readFile(full, 'utf8')).replace(/^/, '') + const parsed = JSON.parse(text) as unknown + if (!Array.isArray(parsed)) throw new Error(`${relPath} is not a JSON array`) + return parsed as T[] +} + +/** Both captures, as the loader and the storefront generator read them. */ +export async function readCaptures(): Promise<{ + avatarItems: AvatarItemCapture[] + skins: SkinCapture[] +}> { + const [avatarItems, skins] = await Promise.all([ + readCapture(AVATAR_ITEMS), + readCapture(SKINS), + ]) + return { avatarItems, skins } +} + +/** Print the duplicate keys a load dropped. Never silent: that is the whole point of them. */ +function reportCollisions(collisions: CatalogCollision[]): void { + if (collisions.length === 0) return + console.warn( + chalk.yellow( + `\n${collisions.length} duplicate item_key(s) in the captures — first occurrence kept:` + ) + ) + const table = new Table({ head: ['item_key', 'kept', 'dropped'] }) + for (const c of collisions) table.push([c.key, c.kept, c.dropped]) + console.warn(table.toString()) + console.warn( + chalk.yellow('A repeat is a defect in the capture — fix the source JSON and load again.\n') + ) +} + +/** + * How many rows the catalog holds, per kind. Read before and after a load: wrangler's meta + * carries no usable row count, so counting the table is the only honest way to say what a load + * actually did. + */ +async function countRows(remote: boolean): Promise> { + const res = await execSql('SELECT kind, COUNT(*) AS n FROM catalog GROUP BY kind', remote, OWNER) + return Object.fromEntries(res.results.map((r) => [String(r.kind), Number(r.n)])) +} + +const total = (counts: Record): number => + Object.values(counts).reduce((a, b) => a + b, 0) + +/** + * The `DO UPDATE` half of the upsert: every column except the conflict target, taken from the + * row that was being inserted. Derived from {@link CATALOG_INSERT_COLUMNS} rather than written + * out, so a column added to the table is carried by a merge automatically — a hand-maintained + * list would quietly stop updating whatever was forgotten. + */ +const CONFLICT_UPDATE = CATALOG_INSERT_COLUMNS.filter((c) => c !== 'item_key') + .map((c) => `${c} = excluded.${c}`) + .join(', ') + +/** + * Prove the load actually landed, and refuse to report success otherwise. + * + * This exists because a load once reported "✓ catalog loaded" over a database it had written + * nothing to. Counting rows back is not paranoia: wrangler's meta carries no usable row count, + * the remote and local paths differ enough that one can break while the other works, and a + * `--file` that fails partway leaves a partial catalog rather than an error. The command must + * either see its own rows in the table or say it failed. + */ +async function verifyLoad( + rows: CatalogLoadRow[], + after: Record, + replace: boolean, + remote: boolean +): Promise { + const got = total(after) + + // A replace empties first, so the count is exact. A merge only adds, so the table must hold at + // least what was just written — more is fine, that is the rows the captures did not mention. + if (replace ? got !== rows.length : got < rows.length) { + throw new Error( + `load did not land: expected ${replace ? '' : 'at least '}${rows.length} rows, ` + + `the table holds ${got}. Nothing was verified as written — re-run it.` + ) + } + + // Counts alone can be satisfied by the rows that were already there, so spot-check actual keys + // from across the file. The last one matters most: a load cut short by a failed batch loses + // the tail while the count still looks plausible. + const probes = [rows[0], rows[Math.floor(rows.length / 2)], rows[rows.length - 1]].filter( + (r): r is CatalogLoadRow => r !== undefined + ) + const list = probes.map((r) => `'${r.key.replaceAll("'", "''")}'`).join(', ') + const res = await execSql( + `SELECT COUNT(*) AS n FROM catalog WHERE item_key IN (${list})`, + remote, + OWNER + ) + const found = Number(res.results[0]?.n ?? 0) + if (found !== probes.length) { + throw new Error( + `load did not land: ${probes.length - found} of ${probes.length} probe rows are missing ` + + `from the table. The catalog may be partially written — re-run it.` + ) + } + + // Every row must have come out with a numeric handle. A NULL here is a load that stopped + // between writing rows and numbering them, which nothing else would notice until a lookup by + // number quietly returned nothing. + const unnumbered = await execSql( + 'SELECT COUNT(*) AS n FROM catalog WHERE catalog_id IS NULL', + remote, + OWNER + ) + const missing = Number(unnumbered.results[0]?.n ?? 0) + if (missing > 0) { + throw new Error(`load did not finish: ${missing} row(s) have no catalog_id. Re-run it.`) + } +} + +const load = new Command('load') + .description('Load the item catalog from the captured JSON (merges by default)') + .option('--local', 'Target the local dev database (the default).', false) + .option('--remote', 'Target the deployed database instead of the local dev database.', false) + .option( + '--replace', + 'Empty the catalog first, so rows absent from the captures are REMOVED.', + false + ) + .option('--dry-run', 'Build and validate the SQL, print what it would do, change nothing.', false) + .action(async (opts) => { + const remote = resolveRemote(opts) + const { avatarItems, skins } = await readCaptures() + const { rows, collisions } = buildCatalogLoad(avatarItems, skins) + + console.log( + `${opts.replace ? 'Replacing the catalog with' : 'Merging'} ${rows.length} rows ` + + `(${avatarItems.length} avatar items, ${skins.length} skins) into ${target(remote)}` + ) + reportCollisions(collisions) + + // MERGE is the default: insert what is new, refresh what already exists, and leave + // anything the captures do not mention alone. That is what makes a PARTIAL capture useful + // — a handful of newly-datamined items can be loaded without having to re-export the whole + // catalog first, and without a partial file silently wiping the rest. + // + // The cost is that a merge can never REMOVE an item: something dropped from the captures + // stays in the table forever. `--replace` is the authoritative reload for when the + // captures are meant to be the whole truth, and it is opt-in because it is the destructive + // one — pointed at a partial capture it would delete everything the file omits. + // + // NO `BEGIN TRANSACTION` / `COMMIT`. Remote D1 REFUSES them outright ("To execute a + // transaction, please use the state.storage.transaction() ... APIs instead of the SQL + // BEGIN TRANSACTION or SAVEPOINT statements"), so a file carrying them loads nothing at + // all against production while working fine locally. Atomicity comes from D1 running each + // batch of statements in its own implicit transaction; across batches there is none, so a + // load that dies midway can leave the catalog partial. That is what {@link verifyLoad} + // below is for, and why re-running is always safe: a merge is idempotent, and a replace + // starts by emptying the table. + const statements: string[] = [] + if (opts.replace) { + statements.push('DELETE FROM catalog;') + } else { + // Clear every existing number BEFORE assigning any. `catalog_id` is unique, and this + // load is about to hand out 1..N: without this, a row already holding one of those + // numbers (because it was in an earlier load and this capture no longer mentions it) + // collides and the whole merge fails. Nulling first also means a row left un-numbered + // afterwards is visibly a row the captures did not mention, which the tail statement + // below then numbers above N. + statements.push('UPDATE catalog SET catalog_id = NULL;') + } + for (let start = 0; start < rows.length; start += CHUNK) { + const batch = rows.slice(start, start + CHUNK) + statements.push( + `INSERT INTO catalog (${CATALOG_INSERT_COLUMNS.join(', ')}) VALUES\n` + + batch.map((r) => `\t(${r.values.map(lit).join(', ')})`).join(',\n') + + // Harmless after a DELETE (nothing can conflict), and kept there anyway so both + // paths run the identical statement rather than two shapes that could diverge. + `\nON CONFLICT(item_key) DO UPDATE SET ${CONFLICT_UPDATE};` + ) + } + if (!opts.replace) { + // Anything still un-numbered is a row the captures did not mention — a merge keeps those, + // so they need handles too. They go ABOVE the highest id this load assigned, so they can + // never collide with it, ordered by rowid for determinism. Usually this matches nothing + // at all, and it is a single statement either way. + const highest = CATALOG_ID_BASE + rows.length - 1 + statements.push( + `UPDATE catalog SET catalog_id = ${highest} +\n` + + `\t(SELECT COUNT(*) FROM catalog c WHERE c.catalog_id IS NULL AND c.rowid <= catalog.rowid)\n` + + `WHERE catalog_id IS NULL;` + ) + } + const sql = statements.join('\n') + + if (opts.dryRun) { + console.log( + chalk.cyan( + `--dry-run: built ${statements.length} statements (${(sql.length / 1024).toFixed(0)} KB); nothing was written.` + ) + ) + return + } + + const before = await countRows(remote) + + // Written to a temp file rather than passed as `--command`: a multi-megabyte argv is not + // something to rely on, and `--file` is the path wrangler batches for us. + const file = path.join(os.tmpdir(), `recflare-catalog-${Date.now()}.sql`) + await fs.writeFile(file, sql) + try { + await execSqlFile(file, remote, OWNER) + } finally { + await fs.remove(file) + } + + const after = await countRows(remote) + const table = new Table({ head: ['kind', 'before', 'after'] }) + for (const kind of new Set([...Object.keys(before), ...Object.keys(after)])) { + table.push([kind, String(before[kind] ?? 0), String(after[kind] ?? 0)]) + } + console.log(table.toString()) + + await verifyLoad(rows, after, opts.replace, remote) + + // On a merge the table only grows, so the growth IS the number of new items and the rest of + // what was written updated a row that already existed. Worth saying out loud: "3370 rows + // loaded" reads like 3370 changes when it may well have been six. + if (!opts.replace) { + const added = total(after) - total(before) + console.log(`${added} new, ${rows.length - added} updated in place, 0 removed`) + } + console.log(chalk.green(`✓ catalog loaded into ${target(remote)}`)) + }) + +const check = new Command('check') + .description('Validate the captured JSON without touching any database') + .action(async () => { + const { avatarItems, skins } = await readCaptures() + const { rows, collisions } = buildCatalogLoad(avatarItems, skins) + + const table = new Table() + table.push( + { 'avatar items': String(avatarItems.length) }, + { skins: String(skins.length) }, + { 'rows to load': String(rows.length) }, + { 'duplicate keys': String(collisions.length) } + ) + console.log(table.toString()) + reportCollisions(collisions) + + // An empty capture is far more likely to be a broken export than a real one, and loading it + // with --replace would swap a good catalog for nothing. + if (rows.length === 0) + throw new Error('the captures produced no rows — refusing to call that ok') + console.log(chalk.green('✓ captures are loadable')) + }) + +export const catalogCmd = new Command('catalog') + .description('Load the item catalog into the shared recflare D1 database') + // Bare `catalog` (no subcommand) prints help and exits cleanly, rather than commander's + // default "missing command" error (exit 1). + .action((_opts, command: Command) => command.outputHelp()) + .addCommand(load) + .addCommand(check) + .addHelpText( + 'after', + ` +The catalog's structure is a migration; its contents are not — reload them here whenever +apps/econ/static/db/*.json changes, with no migration and no deploy. + +load MERGES by default: new items are inserted, existing ones refreshed, and anything the +captures don't mention is left alone — so a partial capture of a few new items is a valid +thing to load. Pass --replace when the captures are the whole truth and items missing from +them should be REMOVED. + +Target --local (default) or --remote (production; needs RECFLARE_D1 in .env). + +Examples: + $ runx catalog check # validate the JSON, touch nothing + $ runx catalog load --dry-run # build the SQL, print what it would do + $ runx catalog load # merge into the local dev database + $ runx catalog load --remote # merge into production + $ runx catalog load --replace # full reload: drops anything not in the captures` + ) diff --git a/packages/tools/src/cmd/storefront.cmd.ts b/packages/tools/src/cmd/storefront.cmd.ts new file mode 100644 index 0000000..a70e4de --- /dev/null +++ b/packages/tools/src/cmd/storefront.cmd.ts @@ -0,0 +1,196 @@ +import 'zx/globals' + +import { writeFileSync } from 'node:fs' +import { Command } from '@commander-js/extra-typings' +import Table from 'cli-table3' + +import { + buildCatalogLoad, + CATALOG_INSERT_COLUMNS, + CatalogKind, + isSellableRarity, + priceForRarity, + subscriberPriceFor, +} from '../../../../apps/econ/src/catalog-load' +import { getRepoRoot } from '../path' +import { readCaptures } from './catalog.cmd' + +/** + * Generate `apps/econ/static/storefronts/sf3-2025.json` — the general store as the 2025 client + * sees it: everything sf3 already sells, PLUS every sellable row of the item catalog. + * + * runx storefront build + * + * One merged file rather than a second storefront id. The client asks for storefront 3 either + * way; the econ worker picks WHICH file by the caller's build (`rn.ver`), so an older build + * keeps exactly the sf3 it has always had and a newer one gets the same store with the + * catalog's items added to it. + * + * A static catalog like every other `sf{N}.json`, because that is the only shape + * `loadStorefront` reads and because browse and BUY have to agree: `findStoreItem` resolves a + * purchase against the very same file, so an item that is not in it cannot be bought. + * + * The two id spaces do not collide, which is what makes the merge safe: every captured sf3 id + * is 2764 or below (one outlier aside) and every catalog id starts at `CATALOG_ID_BASE` + * (10000). Nothing is renumbered, and an id means the same item in both files. + */ + +const OUT_DIR = 'apps/econ/static/storefronts' + +/** The captured general store, whose items the generated one is built on top of. */ +const BASE_STOREFRONT = `${OUT_DIR}/sf3.json` + +/** The merged store the newer client is served, and the `StorefrontType` it reports. */ +const OUT_FILE = `${OUT_DIR}/sf3-2025.json` +const STOREFRONT_TYPE = 3 + +/** RecCenterTokens — the currency the avatar storefronts sell in. */ +const CURRENCY_TYPE_TOKENS = 2 + +/** + * Far enough out that the client never refetches — the same sentinel sf3 carries. A real + * storefront rotates; this one is regenerated by hand, so it must not expire on its own. + */ +const NEXT_UPDATE = '2226-06-14T00:12:20.1324853Z' + +const build = new Command('build') + .description('Generate the merged 2025 general store (sf3 + the item catalog)') + .action(async () => { + const { avatarItems, skins } = await readCaptures() + + // sf3's own items, carried through UNCHANGED. They keep their ids, their prices and their + // gift-drops: the merge adds to the store the older client knows, it does not restate it. + const basePath = path.join(getRepoRoot(), BASE_STOREFRONT) + const base = JSON.parse(await fs.readFile(basePath, 'utf8')) as { + StoreItems: Array<{ PurchasableItemId: number }> + } + + // The catalog ids come from the same loader the DB load uses, so the number in this file is + // the number in the table. An avatar item's `item_key` IS its `AvatarItemDesc`, which is + // what lets the two be matched up without a second numbering scheme to keep in step. + const { rows } = buildCatalogLoad(avatarItems, skins) + const kindAt = CATALOG_INSERT_COLUMNS.indexOf('kind') + const catalogIdByDesc = new Map( + rows.filter((r) => r.values[kindAt] === CatalogKind.AvatarItem).map((r) => [r.key, r.id]) + ) + + const byRarity = new Map() + const excluded = new Map() + const forSale = avatarItems.filter((item) => { + if (isSellableRarity(item.Rarity)) return true + excluded.set(item.Rarity, (excluded.get(item.Rarity) ?? 0) + 1) + return false + }) + const storeItems = forSale.map((item) => { + const catalogId = catalogIdByDesc.get(item.AvatarItemDesc) + if (catalogId === undefined) { + throw new Error(`no catalog id for avatar item ${item.AvatarItemDesc}`) + } + const price = priceForRarity(item.Rarity) + byRarity.set(item.Rarity, (byRarity.get(item.Rarity) ?? 0) + 1) + const priceEntry = (p: number) => [ + { CurrencyType: CURRENCY_TYPE_TOKENS, Price: p, StorefrontSaleData: null, Type: 0 }, + ] + return { + // Key order and every constant field mirror sf3, because the client's parser reads + // this shape and an sf that differs from the one known to work is a needless variable. + GiftDrop: { + AvatarItemDesc: item.AvatarItemDesc, + AvatarItemType: item.AvatarItemType, + ConsumableItemDesc: '', + Context: 0, + Currency: 0, + CurrencyType: 0, + EquipmentModificationGuid: '', + EquipmentPrefabName: '', + FriendlyName: item.FriendlyName, + // sf3 has GiftDropId === PurchasableItemId on all 1161 of its items; keep that. + GiftDropId: catalogId, + IsQuery: false, + ItemSetFriendlyName: '', + ItemSetId: 0, + Rarity: item.Rarity, + SubscribersOnly: false, + // The catalog's tooltip is genuinely null on some rows; the client's field is a + // string, so this is the one place the distinction is flattened. + Tooltip: item.Tooltip ?? '', + Unique: true, + }, + IsFeatured: false, + Prices: priceEntry(price), + // The catalog id, directly — one number, no second numbering to keep in step. It is + // already clear of every captured storefront's ids (see `CATALOG_ID_BASE`), which is + // what lets it be used here as-is. + PurchasableItemId: catalogId, + SubscriberPrices: priceEntry(subscriberPriceFor(price)), + Type: 0, + } + }) + + // An id colliding across the two would make one number mean two different items depending + // on which half answered first — refused rather than resolved by ordering, since the whole + // point of `CATALOG_ID_BASE` is that this cannot happen. + const baseIds = new Set(base.StoreItems.map((i) => i.PurchasableItemId)) + const collisions = storeItems.filter((i) => baseIds.has(i.PurchasableItemId)) + if (collisions.length > 0) { + throw new Error( + `${collisions.length} catalog id(s) collide with sf3's own, starting at ` + + `${collisions[0]?.PurchasableItemId}. The catalog must be renumbered above them.` + ) + } + + const storefront = { + NextUpdate: NEXT_UPDATE, + StoreItems: [...base.StoreItems, ...storeItems], + StorefrontType: STOREFRONT_TYPE, + // Deliberately 0 — the discount is expressed ONLY in `SubscriberPrices`. Announcing it + // again here risks a client taking the 10% off an already-discounted price and posting + // through the server's own subscriber floor, refused as "Price has changed". + SubscriberDiscountPercent: 0, + } + + writeFileSync(OUT_FILE, `${JSON.stringify(storefront, null, '\t')}\n`) + + const table = new Table({ head: ['rarity', 'items', 'price', 'subscriber'] }) + for (const rarity of [...byRarity.keys()].sort((a, b) => a - b)) { + const price = priceForRarity(rarity) + table.push([ + String(rarity), + String(byRarity.get(rarity)), + String(price), + String(subscriberPriceFor(price)), + ]) + } + console.log(table.toString()) + for (const [rarity, n] of [...excluded].sort((a, b) => a[0] - b[0])) { + console.log(chalk.yellow(`excluded ${n} item(s) of rarity ${rarity} — not for sale`)) + } + console.log( + chalk.green( + `✓ wrote ${OUT_FILE}: ${storefront.StoreItems.length} items ` + + `(${base.StoreItems.length} from sf3 + ${storeItems.length} from the catalog, ` + + `${((await fs.stat(OUT_FILE)).size / 1024 / 1024).toFixed(1)} MB)` + ) + ) + }) + +export const storefrontCmd = new Command('storefront') + .description('Generate the storefront catalogs the econ worker serves') + // Bare `storefront` (no subcommand) prints help and exits cleanly, rather than commander's + // default "missing command" error (exit 1). + .action((_opts, command: Command) => command.outputHelp()) + .addCommand(build) + .addHelpText( + 'after', + ` +sf3-2025 is the general store as the 2025 client sees it: everything sf3 already sells plus +every sellable row of the item catalog, priced by rarity. The client asks for storefront 3 +either way — the econ worker picks which file by the caller's build, so an older client keeps +the sf3 it has always had. + +Regenerate whenever apps/econ/static/db/*.json changes, and after every \`runx catalog load\`: +a load renumbers catalog_id, and a stale file would list the wrong items. + +Examples: + $ runx storefront build # write apps/econ/static/storefronts/sf3-2025.json` + ) diff --git a/packages/tools/src/d1.ts b/packages/tools/src/d1.ts new file mode 100644 index 0000000..7899010 --- /dev/null +++ b/packages/tools/src/d1.ts @@ -0,0 +1,122 @@ +import 'zx/globals' + +import { getRepoRoot } from './path' + +/** + * Shared plumbing for talking to the one `recflare` D1 database from a CLI. + * + * Everything here shells out to `wrangler d1 execute` rather than calling a running worker, + * so no deploy, no auth token and no Secrets Store read is involved — the last of which is + * not even possible, since a Secrets Store value is write-only outside a bound Worker. + * + * `--local` (the default) hits the local dev database; `--remote` hits the deployed one and + * needs the real D1 id, because the committed `wrangler.jsonc` files all carry the literal + * placeholder `"local"` as their `database_id`. Splicing the real id into a gitignored + * generated config is the same thing `run-wrangler-migrate` does at deploy time. + */ + +/** The one shared database every D1-backed worker binds. */ +export const DB_NAME = 'recflare' + +export interface D1ExecResult { + results: Array> + success: boolean + meta: { changes?: number; rows_read?: number } +} + +/** Escape a value for embedding inside a single-quoted SQL string literal. */ +export const sqlStr = (s: string): string => s.replace(/'/g, "''") + +/** A short, loud label for which database a command is about to touch. */ +export const target = (remote: boolean): string => + remote ? chalk.red(`${DB_NAME} (remote)`) : chalk.cyan(`${DB_NAME} (local)`) + +/** `--local` is the default; passing both is a mistake worth refusing rather than guessing. */ +export function resolveRemote(opts: { local?: boolean; remote?: boolean }): boolean { + if (opts.local && opts.remote) throw new Error('pass at most one of --local / --remote') + return opts.remote === true +} + +/** The deployed D1's real id, from the environment or the gitignored root .env. */ +export async function getRemoteD1Id(): Promise { + if (process.env.RECFLARE_D1) return process.env.RECFLARE_D1 + const envPath = path.join(getRepoRoot(), '.env') + if (await fs.pathExists(envPath)) { + const content = await fs.readFile(envPath, 'utf8') + const m = content.match(/^\s*RECFLARE_D1\s*=\s*(.+?)\s*$/m) + if (m) return m[1].replace(/^["']|["']$/g, '') + } + throw new Error('RECFLARE_D1 is not set — add the recflare D1 id to .env (see .env.example)') +} + +/** + * Run wrangler `d1 execute` from one worker's directory, with the extra args a caller + * supplies (`--command …` or `--file …`). + * + * The worker only decides which `wrangler.jsonc` is read and whose `node_modules` wrangler + * resolves from — the database is the same one either way. Pick the worker that OWNS the + * table being touched, so the config that gets spliced is the one whose migrations built it. + */ +async function runD1(worker: string, extraArgs: string[], remote: boolean): Promise { + const workerDir = path.join(getRepoRoot(), 'apps', worker) + cd(workerDir) + + const args = ['d1', 'execute', DB_NAME, ...extraArgs] + let cleanup: (() => Promise) | undefined + + if (remote) { + const id = await getRemoteD1Id() + const src = await fs.readFile(path.join(workerDir, 'wrangler.jsonc'), 'utf8') + const generated = src.replace(/("database_id"\s*:\s*")[^"]*(")/, `$1${id}$2`) + const genPath = path.join(workerDir, 'wrangler.generated.jsonc') + await fs.writeFile(genPath, generated) + cleanup = () => fs.remove(genPath) + // `--yes` because wrangler asks for confirmation before touching the deployed database + // and a swallowed prompt looks exactly like a hang. + args.push('--config', 'wrangler.generated.jsonc', '--remote', '--yes') + } else { + args.push('--local') + } + + try { + // Via `pnpm exec` so wrangler resolves from the worker's node_modules (it isn't a + // dependency of @repo/tools, so it's not on this process's PATH). + const out = await $`pnpm exec wrangler ${args}`.quiet() + return out.stdout + } finally { + if (cleanup) await cleanup() + } +} + +/** + * Run one SQL statement and parse the result. + * + * Read the RESULTS to find out what happened, not the meta: wrangler's `--json` meta carries + * only a duration, with no reliable `changes` count, so a statement that needs to know what it + * matched has to say `RETURNING`. + */ +export async function execSql( + sql: string, + remote: boolean, + worker = 'auth' +): Promise { + const stdout = await runD1(worker, ['--command', sql, '--json'], remote) + // wrangler --json prints a one-element array of results to stdout. + const start = stdout.indexOf('[') + if (start === -1) throw new Error(`unexpected d1 execute output:\n${stdout}`) + const parsed = JSON.parse(stdout.slice(start)) as D1ExecResult[] + const first = parsed[0] + if (!first) throw new Error(`empty d1 execute result:\n${stdout}`) + return first +} + +/** + * Ingest a whole `.sql` file. Used for bulk loads far too large for `--command`, where + * wrangler splits the file into statements and batches them itself. + * + * Not `--json`: a multi-thousand-statement file answers with a result object per statement, + * which is megabytes of nothing. The human-readable output is returned for the caller. + */ +export async function execSqlFile(file: string, remote: boolean, worker = 'auth'): Promise { + return await runD1(worker, ['--file', file], remote) +}