mirror of
https://github.com/djdevin/recflare.git
synced 2026-09-08 06:31:27 -07:00
[packages] tools for catalog building
This commit is contained in:
Devin Zuczek
@@ -5,11 +5,13 @@ import { catchProcessError } from '@jahands/cli-tools/proc'
|
||||
|
||||
import { adminCmd } from '../cmd/admin.cmd'
|
||||
import { buildCmd } from '../cmd/build.cmd'
|
||||
import { catalogCmd } from '../cmd/catalog.cmd'
|
||||
import { checkCmd } from '../cmd/check.cmd'
|
||||
import { ciCmd } from '../cmd/ci.cmd'
|
||||
import { devCmd } from '../cmd/dev.cmd'
|
||||
import { fixCmd } from '../cmd/fix.cmd'
|
||||
import { shfmtCmd } from '../cmd/shfmt.cmd'
|
||||
import { storefrontCmd } from '../cmd/storefront.cmd'
|
||||
import { updateCmd } from '../cmd/update.cmd'
|
||||
|
||||
program
|
||||
@@ -20,6 +22,8 @@ program
|
||||
// a typescript CLI is nicer for more complex things.
|
||||
|
||||
.addCommand(adminCmd)
|
||||
.addCommand(catalogCmd)
|
||||
.addCommand(storefrontCmd)
|
||||
.addCommand(fixCmd)
|
||||
.addCommand(buildCmd)
|
||||
.addCommand(checkCmd)
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import * as readline from 'node:readline'
|
||||
|
||||
import { Command } from '@commander-js/extra-typings'
|
||||
import Table from 'cli-table3'
|
||||
|
||||
import { getRepoRoot } from '../path'
|
||||
import { execSql, resolveRemote, sqlStr, target } from '../d1'
|
||||
import { hashPassword } from '../password'
|
||||
|
||||
import type { D1ExecResult } from '../d1'
|
||||
|
||||
/**
|
||||
* Operator-facing admin tools for the shared `recflare` D1 database. Each command
|
||||
* shells out to `wrangler d1 execute recflare` — no running worker or auth token
|
||||
@@ -19,18 +20,6 @@ import { hashPassword } from '../password'
|
||||
* runx admin grant-developer --account 1 [--revoke] [--remote]
|
||||
*/
|
||||
|
||||
/** The one shared database every D1-backed worker binds. */
|
||||
const DB_NAME = 'recflare'
|
||||
|
||||
interface D1ExecResult {
|
||||
results: Array<Record<string, unknown>>
|
||||
success: boolean
|
||||
meta: { changes?: number; rows_read?: number }
|
||||
}
|
||||
|
||||
/** Escape a value for embedding inside a single-quoted SQL string literal. */
|
||||
const sqlStr = (s: string): string => s.replace(/'/g, "''")
|
||||
|
||||
/**
|
||||
* Resolve the account selector into a SQL WHERE fragment. Exactly one of
|
||||
* `--account` / `--username` must be given. Account ids are validated numeric;
|
||||
@@ -50,59 +39,6 @@ function whereClause(account?: string, username?: string): { where: string; labe
|
||||
}
|
||||
}
|
||||
|
||||
/** The deployed D1's real id, from the environment or the gitignored root .env. */
|
||||
async function getRemoteD1Id(): Promise<string> {
|
||||
if (process.env.RECFLARE_D1) return process.env.RECFLARE_D1
|
||||
const envPath = path.join(getRepoRoot(), '.env')
|
||||
if (await fs.pathExists(envPath)) {
|
||||
const content = await fs.readFile(envPath, 'utf8')
|
||||
const m = content.match(/^\s*RECFLARE_D1\s*=\s*(.+?)\s*$/m)
|
||||
if (m) return m[1].replace(/^["']|["']$/g, '')
|
||||
}
|
||||
throw new Error('RECFLARE_D1 is not set — add the recflare D1 id to .env (see .env.example)')
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a SQL statement against the shared database via wrangler. Runs from the auth
|
||||
* worker's directory (it owns the accounts schema and binds the DB). For `--remote`
|
||||
* the committed wrangler.jsonc's "local" database_id placeholder is spliced with the
|
||||
* real id into a gitignored generated config — exactly like run-wrangler-migrate.
|
||||
*/
|
||||
async function execSql(sql: string, remote: boolean): Promise<D1ExecResult> {
|
||||
const authDir = path.join(getRepoRoot(), 'apps', 'auth')
|
||||
cd(authDir)
|
||||
|
||||
const args = ['d1', 'execute', DB_NAME, '--command', sql, '--json']
|
||||
let cleanup: (() => Promise<void>) | undefined
|
||||
|
||||
if (remote) {
|
||||
const id = await getRemoteD1Id()
|
||||
const src = await fs.readFile(path.join(authDir, 'wrangler.jsonc'), 'utf8')
|
||||
const generated = src.replace(/("database_id"\s*:\s*")[^"]*(")/, `$1${id}$2`)
|
||||
const genPath = path.join(authDir, 'wrangler.generated.jsonc')
|
||||
await fs.writeFile(genPath, generated)
|
||||
cleanup = () => fs.remove(genPath)
|
||||
args.push('--config', 'wrangler.generated.jsonc', '--remote')
|
||||
} else {
|
||||
args.push('--local')
|
||||
}
|
||||
|
||||
try {
|
||||
// Via `pnpm exec` so wrangler resolves from the auth worker's node_modules
|
||||
// (it isn't a dependency of @repo/tools, so it's not on this process's PATH).
|
||||
const out = await $`pnpm exec wrangler ${args}`.quiet()
|
||||
// wrangler --json prints a one-element array of results to stdout.
|
||||
const start = out.stdout.indexOf('[')
|
||||
if (start === -1) throw new Error(`unexpected d1 execute output:\n${out.stdout}`)
|
||||
const parsed = JSON.parse(out.stdout.slice(start)) as D1ExecResult[]
|
||||
const first = parsed[0]
|
||||
if (!first) throw new Error(`empty d1 execute result:\n${out.stdout}`)
|
||||
return first
|
||||
} finally {
|
||||
if (cleanup) await cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
/** Prompt for a line of input without echoing what's typed (for passwords). */
|
||||
function promptHidden(query: string): Promise<string> {
|
||||
return new Promise((resolve) => {
|
||||
@@ -131,7 +67,9 @@ async function resolvePassword(flag?: string): Promise<string> {
|
||||
if (!process.stdin.isTTY) {
|
||||
const chunks: Buffer[] = []
|
||||
for await (const chunk of process.stdin) chunks.push(chunk as Buffer)
|
||||
const piped = Buffer.concat(chunks).toString('utf8').replace(/\r?\n$/, '')
|
||||
const piped = Buffer.concat(chunks)
|
||||
.toString('utf8')
|
||||
.replace(/\r?\n$/, '')
|
||||
if (piped === '') throw new Error('no password provided on stdin')
|
||||
return piped
|
||||
}
|
||||
@@ -142,16 +80,6 @@ async function resolvePassword(flag?: string): Promise<string> {
|
||||
return first
|
||||
}
|
||||
|
||||
/** A short, loud label for which database a command is about to touch. */
|
||||
const target = (remote: boolean): string =>
|
||||
remote ? chalk.red(`${DB_NAME} (remote)`) : chalk.cyan(`${DB_NAME} (local)`)
|
||||
|
||||
/** Resolve the --local/--remote target flags. Local is the default. */
|
||||
function resolveRemote(opts: { local?: boolean; remote?: boolean }): boolean {
|
||||
if (opts.local && opts.remote) throw new Error('pass at most one of --local / --remote')
|
||||
return opts.remote === true
|
||||
}
|
||||
|
||||
/**
|
||||
* Fail when a WHERE-scoped UPDATE matched no row (i.e. no such account). Relies on
|
||||
* the statement's `RETURNING account_id` — wrangler's `--json` meta doesn't reliably
|
||||
@@ -251,7 +179,11 @@ const lookup = new Command('lookup')
|
||||
return
|
||||
}
|
||||
const asText = (v: unknown): string =>
|
||||
v == null ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v as number | string | boolean)
|
||||
v == null
|
||||
? ''
|
||||
: typeof v === 'object'
|
||||
? JSON.stringify(v)
|
||||
: String(v as number | string | boolean)
|
||||
const boolKeys = new Set(['hasPassword', 'isDeveloper', 'isModerator'])
|
||||
const table = new Table()
|
||||
for (const [key, value] of Object.entries(row)) {
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
import { Command } from '@commander-js/extra-typings'
|
||||
import Table from 'cli-table3'
|
||||
|
||||
import {
|
||||
buildCatalogLoad,
|
||||
CATALOG_ID_BASE,
|
||||
CATALOG_INSERT_COLUMNS,
|
||||
} from '../../../../apps/econ/src/catalog-load'
|
||||
import { execSql, execSqlFile, resolveRemote, target } from '../d1'
|
||||
import { getRepoRoot } from '../path'
|
||||
|
||||
import type {
|
||||
AvatarItemCapture,
|
||||
CatalogCollision,
|
||||
CatalogLoadRow,
|
||||
CatalogValue,
|
||||
SkinCapture,
|
||||
} from '../../../../apps/econ/src/catalog-load'
|
||||
|
||||
/**
|
||||
* Load the item catalog into the shared `recflare` D1 database.
|
||||
*
|
||||
* The `catalog` table's STRUCTURE is a migration (apps/econ/migrations/0015_catalog.sql); its
|
||||
* CONTENTS are not. The game's item list changes without the schema changing, so shipping the
|
||||
* rows as a migration would mean a migration and a deploy per refresh, an ever-growing pile of
|
||||
* near-identical data migrations, and no way to reload without writing another one. This
|
||||
* command is the reload.
|
||||
*
|
||||
* runx catalog load [--remote] [--replace] [--dry-run]
|
||||
* runx catalog check
|
||||
*
|
||||
* The mapping from capture to row lives in the econ worker's `catalog-load.ts`, beside the
|
||||
* schema it fills, so a column added there and a column added to the loader cannot drift. That
|
||||
* module deliberately touches no Workers types, which is what lets this Node CLI import it.
|
||||
*/
|
||||
|
||||
/** The worker that owns the table — whose wrangler.jsonc and node_modules wrangler uses. */
|
||||
const OWNER = 'econ'
|
||||
|
||||
const AVATAR_ITEMS = 'apps/econ/static/db/avatar-items.json'
|
||||
const SKINS = 'apps/econ/static/db/skins.json'
|
||||
|
||||
/**
|
||||
* Rows per INSERT. Batched because thousands of single-row statements parse far more slowly
|
||||
* than a few dozen multi-row ones, and D1 charges per statement on the remote path.
|
||||
*/
|
||||
const CHUNK = 100
|
||||
|
||||
/**
|
||||
* One SQL literal. Booleans become 1/0 (SQLite has no boolean), `undefined` is treated as the
|
||||
* NULL it stands for — a key the capture omitted rather than set — and quotes are doubled.
|
||||
* Item names are the only free text here, but escaping is what makes running this against a
|
||||
* fresh capture safe rather than lucky.
|
||||
*/
|
||||
function lit(v: CatalogValue): string {
|
||||
if (v === null || v === undefined) return 'NULL'
|
||||
if (typeof v === 'boolean') return v ? '1' : '0'
|
||||
if (typeof v === 'number') return String(v)
|
||||
return `'${v.replaceAll("'", "''")}'`
|
||||
}
|
||||
|
||||
/** Read one capture, tolerating the BOM the exports carry (`JSON.parse` rejects U+FEFF). */
|
||||
async function readCapture<T>(relPath: string): Promise<T[]> {
|
||||
const full = path.join(getRepoRoot(), relPath)
|
||||
if (!(await fs.pathExists(full))) throw new Error(`no capture at ${relPath}`)
|
||||
const text = (await fs.readFile(full, 'utf8')).replace(/^/, '')
|
||||
const parsed = JSON.parse(text) as unknown
|
||||
if (!Array.isArray(parsed)) throw new Error(`${relPath} is not a JSON array`)
|
||||
return parsed as T[]
|
||||
}
|
||||
|
||||
/** Both captures, as the loader and the storefront generator read them. */
|
||||
export async function readCaptures(): Promise<{
|
||||
avatarItems: AvatarItemCapture[]
|
||||
skins: SkinCapture[]
|
||||
}> {
|
||||
const [avatarItems, skins] = await Promise.all([
|
||||
readCapture<AvatarItemCapture>(AVATAR_ITEMS),
|
||||
readCapture<SkinCapture>(SKINS),
|
||||
])
|
||||
return { avatarItems, skins }
|
||||
}
|
||||
|
||||
/** Print the duplicate keys a load dropped. Never silent: that is the whole point of them. */
|
||||
function reportCollisions(collisions: CatalogCollision[]): void {
|
||||
if (collisions.length === 0) return
|
||||
console.warn(
|
||||
chalk.yellow(
|
||||
`\n${collisions.length} duplicate item_key(s) in the captures — first occurrence kept:`
|
||||
)
|
||||
)
|
||||
const table = new Table({ head: ['item_key', 'kept', 'dropped'] })
|
||||
for (const c of collisions) table.push([c.key, c.kept, c.dropped])
|
||||
console.warn(table.toString())
|
||||
console.warn(
|
||||
chalk.yellow('A repeat is a defect in the capture — fix the source JSON and load again.\n')
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* How many rows the catalog holds, per kind. Read before and after a load: wrangler's meta
|
||||
* carries no usable row count, so counting the table is the only honest way to say what a load
|
||||
* actually did.
|
||||
*/
|
||||
async function countRows(remote: boolean): Promise<Record<string, number>> {
|
||||
const res = await execSql('SELECT kind, COUNT(*) AS n FROM catalog GROUP BY kind', remote, OWNER)
|
||||
return Object.fromEntries(res.results.map((r) => [String(r.kind), Number(r.n)]))
|
||||
}
|
||||
|
||||
const total = (counts: Record<string, number>): number =>
|
||||
Object.values(counts).reduce((a, b) => a + b, 0)
|
||||
|
||||
/**
|
||||
* The `DO UPDATE` half of the upsert: every column except the conflict target, taken from the
|
||||
* row that was being inserted. Derived from {@link CATALOG_INSERT_COLUMNS} rather than written
|
||||
* out, so a column added to the table is carried by a merge automatically — a hand-maintained
|
||||
* list would quietly stop updating whatever was forgotten.
|
||||
*/
|
||||
const CONFLICT_UPDATE = CATALOG_INSERT_COLUMNS.filter((c) => c !== 'item_key')
|
||||
.map((c) => `${c} = excluded.${c}`)
|
||||
.join(', ')
|
||||
|
||||
/**
|
||||
* Prove the load actually landed, and refuse to report success otherwise.
|
||||
*
|
||||
* This exists because a load once reported "✓ catalog loaded" over a database it had written
|
||||
* nothing to. Counting rows back is not paranoia: wrangler's meta carries no usable row count,
|
||||
* the remote and local paths differ enough that one can break while the other works, and a
|
||||
* `--file` that fails partway leaves a partial catalog rather than an error. The command must
|
||||
* either see its own rows in the table or say it failed.
|
||||
*/
|
||||
async function verifyLoad(
|
||||
rows: CatalogLoadRow[],
|
||||
after: Record<string, number>,
|
||||
replace: boolean,
|
||||
remote: boolean
|
||||
): Promise<void> {
|
||||
const got = total(after)
|
||||
|
||||
// A replace empties first, so the count is exact. A merge only adds, so the table must hold at
|
||||
// least what was just written — more is fine, that is the rows the captures did not mention.
|
||||
if (replace ? got !== rows.length : got < rows.length) {
|
||||
throw new Error(
|
||||
`load did not land: expected ${replace ? '' : 'at least '}${rows.length} rows, ` +
|
||||
`the table holds ${got}. Nothing was verified as written — re-run it.`
|
||||
)
|
||||
}
|
||||
|
||||
// Counts alone can be satisfied by the rows that were already there, so spot-check actual keys
|
||||
// from across the file. The last one matters most: a load cut short by a failed batch loses
|
||||
// the tail while the count still looks plausible.
|
||||
const probes = [rows[0], rows[Math.floor(rows.length / 2)], rows[rows.length - 1]].filter(
|
||||
(r): r is CatalogLoadRow => r !== undefined
|
||||
)
|
||||
const list = probes.map((r) => `'${r.key.replaceAll("'", "''")}'`).join(', ')
|
||||
const res = await execSql(
|
||||
`SELECT COUNT(*) AS n FROM catalog WHERE item_key IN (${list})`,
|
||||
remote,
|
||||
OWNER
|
||||
)
|
||||
const found = Number(res.results[0]?.n ?? 0)
|
||||
if (found !== probes.length) {
|
||||
throw new Error(
|
||||
`load did not land: ${probes.length - found} of ${probes.length} probe rows are missing ` +
|
||||
`from the table. The catalog may be partially written — re-run it.`
|
||||
)
|
||||
}
|
||||
|
||||
// Every row must have come out with a numeric handle. A NULL here is a load that stopped
|
||||
// between writing rows and numbering them, which nothing else would notice until a lookup by
|
||||
// number quietly returned nothing.
|
||||
const unnumbered = await execSql(
|
||||
'SELECT COUNT(*) AS n FROM catalog WHERE catalog_id IS NULL',
|
||||
remote,
|
||||
OWNER
|
||||
)
|
||||
const missing = Number(unnumbered.results[0]?.n ?? 0)
|
||||
if (missing > 0) {
|
||||
throw new Error(`load did not finish: ${missing} row(s) have no catalog_id. Re-run it.`)
|
||||
}
|
||||
}
|
||||
|
||||
const load = new Command('load')
|
||||
.description('Load the item catalog from the captured JSON (merges by default)')
|
||||
.option('--local', 'Target the local dev database (the default).', false)
|
||||
.option('--remote', 'Target the deployed database instead of the local dev database.', false)
|
||||
.option(
|
||||
'--replace',
|
||||
'Empty the catalog first, so rows absent from the captures are REMOVED.',
|
||||
false
|
||||
)
|
||||
.option('--dry-run', 'Build and validate the SQL, print what it would do, change nothing.', false)
|
||||
.action(async (opts) => {
|
||||
const remote = resolveRemote(opts)
|
||||
const { avatarItems, skins } = await readCaptures()
|
||||
const { rows, collisions } = buildCatalogLoad(avatarItems, skins)
|
||||
|
||||
console.log(
|
||||
`${opts.replace ? 'Replacing the catalog with' : 'Merging'} ${rows.length} rows ` +
|
||||
`(${avatarItems.length} avatar items, ${skins.length} skins) into ${target(remote)}`
|
||||
)
|
||||
reportCollisions(collisions)
|
||||
|
||||
// MERGE is the default: insert what is new, refresh what already exists, and leave
|
||||
// anything the captures do not mention alone. That is what makes a PARTIAL capture useful
|
||||
// — a handful of newly-datamined items can be loaded without having to re-export the whole
|
||||
// catalog first, and without a partial file silently wiping the rest.
|
||||
//
|
||||
// The cost is that a merge can never REMOVE an item: something dropped from the captures
|
||||
// stays in the table forever. `--replace` is the authoritative reload for when the
|
||||
// captures are meant to be the whole truth, and it is opt-in because it is the destructive
|
||||
// one — pointed at a partial capture it would delete everything the file omits.
|
||||
//
|
||||
// NO `BEGIN TRANSACTION` / `COMMIT`. Remote D1 REFUSES them outright ("To execute a
|
||||
// transaction, please use the state.storage.transaction() ... APIs instead of the SQL
|
||||
// BEGIN TRANSACTION or SAVEPOINT statements"), so a file carrying them loads nothing at
|
||||
// all against production while working fine locally. Atomicity comes from D1 running each
|
||||
// batch of statements in its own implicit transaction; across batches there is none, so a
|
||||
// load that dies midway can leave the catalog partial. That is what {@link verifyLoad}
|
||||
// below is for, and why re-running is always safe: a merge is idempotent, and a replace
|
||||
// starts by emptying the table.
|
||||
const statements: string[] = []
|
||||
if (opts.replace) {
|
||||
statements.push('DELETE FROM catalog;')
|
||||
} else {
|
||||
// Clear every existing number BEFORE assigning any. `catalog_id` is unique, and this
|
||||
// load is about to hand out 1..N: without this, a row already holding one of those
|
||||
// numbers (because it was in an earlier load and this capture no longer mentions it)
|
||||
// collides and the whole merge fails. Nulling first also means a row left un-numbered
|
||||
// afterwards is visibly a row the captures did not mention, which the tail statement
|
||||
// below then numbers above N.
|
||||
statements.push('UPDATE catalog SET catalog_id = NULL;')
|
||||
}
|
||||
for (let start = 0; start < rows.length; start += CHUNK) {
|
||||
const batch = rows.slice(start, start + CHUNK)
|
||||
statements.push(
|
||||
`INSERT INTO catalog (${CATALOG_INSERT_COLUMNS.join(', ')}) VALUES\n` +
|
||||
batch.map((r) => `\t(${r.values.map(lit).join(', ')})`).join(',\n') +
|
||||
// Harmless after a DELETE (nothing can conflict), and kept there anyway so both
|
||||
// paths run the identical statement rather than two shapes that could diverge.
|
||||
`\nON CONFLICT(item_key) DO UPDATE SET ${CONFLICT_UPDATE};`
|
||||
)
|
||||
}
|
||||
if (!opts.replace) {
|
||||
// Anything still un-numbered is a row the captures did not mention — a merge keeps those,
|
||||
// so they need handles too. They go ABOVE the highest id this load assigned, so they can
|
||||
// never collide with it, ordered by rowid for determinism. Usually this matches nothing
|
||||
// at all, and it is a single statement either way.
|
||||
const highest = CATALOG_ID_BASE + rows.length - 1
|
||||
statements.push(
|
||||
`UPDATE catalog SET catalog_id = ${highest} +\n` +
|
||||
`\t(SELECT COUNT(*) FROM catalog c WHERE c.catalog_id IS NULL AND c.rowid <= catalog.rowid)\n` +
|
||||
`WHERE catalog_id IS NULL;`
|
||||
)
|
||||
}
|
||||
const sql = statements.join('\n')
|
||||
|
||||
if (opts.dryRun) {
|
||||
console.log(
|
||||
chalk.cyan(
|
||||
`--dry-run: built ${statements.length} statements (${(sql.length / 1024).toFixed(0)} KB); nothing was written.`
|
||||
)
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
const before = await countRows(remote)
|
||||
|
||||
// Written to a temp file rather than passed as `--command`: a multi-megabyte argv is not
|
||||
// something to rely on, and `--file` is the path wrangler batches for us.
|
||||
const file = path.join(os.tmpdir(), `recflare-catalog-${Date.now()}.sql`)
|
||||
await fs.writeFile(file, sql)
|
||||
try {
|
||||
await execSqlFile(file, remote, OWNER)
|
||||
} finally {
|
||||
await fs.remove(file)
|
||||
}
|
||||
|
||||
const after = await countRows(remote)
|
||||
const table = new Table({ head: ['kind', 'before', 'after'] })
|
||||
for (const kind of new Set([...Object.keys(before), ...Object.keys(after)])) {
|
||||
table.push([kind, String(before[kind] ?? 0), String(after[kind] ?? 0)])
|
||||
}
|
||||
console.log(table.toString())
|
||||
|
||||
await verifyLoad(rows, after, opts.replace, remote)
|
||||
|
||||
// On a merge the table only grows, so the growth IS the number of new items and the rest of
|
||||
// what was written updated a row that already existed. Worth saying out loud: "3370 rows
|
||||
// loaded" reads like 3370 changes when it may well have been six.
|
||||
if (!opts.replace) {
|
||||
const added = total(after) - total(before)
|
||||
console.log(`${added} new, ${rows.length - added} updated in place, 0 removed`)
|
||||
}
|
||||
console.log(chalk.green(`✓ catalog loaded into ${target(remote)}`))
|
||||
})
|
||||
|
||||
const check = new Command('check')
|
||||
.description('Validate the captured JSON without touching any database')
|
||||
.action(async () => {
|
||||
const { avatarItems, skins } = await readCaptures()
|
||||
const { rows, collisions } = buildCatalogLoad(avatarItems, skins)
|
||||
|
||||
const table = new Table()
|
||||
table.push(
|
||||
{ 'avatar items': String(avatarItems.length) },
|
||||
{ skins: String(skins.length) },
|
||||
{ 'rows to load': String(rows.length) },
|
||||
{ 'duplicate keys': String(collisions.length) }
|
||||
)
|
||||
console.log(table.toString())
|
||||
reportCollisions(collisions)
|
||||
|
||||
// An empty capture is far more likely to be a broken export than a real one, and loading it
|
||||
// with --replace would swap a good catalog for nothing.
|
||||
if (rows.length === 0)
|
||||
throw new Error('the captures produced no rows — refusing to call that ok')
|
||||
console.log(chalk.green('✓ captures are loadable'))
|
||||
})
|
||||
|
||||
export const catalogCmd = new Command('catalog')
|
||||
.description('Load the item catalog into the shared recflare D1 database')
|
||||
// Bare `catalog` (no subcommand) prints help and exits cleanly, rather than commander's
|
||||
// default "missing command" error (exit 1).
|
||||
.action((_opts, command: Command) => command.outputHelp())
|
||||
.addCommand(load)
|
||||
.addCommand(check)
|
||||
.addHelpText(
|
||||
'after',
|
||||
`
|
||||
The catalog's structure is a migration; its contents are not — reload them here whenever
|
||||
apps/econ/static/db/*.json changes, with no migration and no deploy.
|
||||
|
||||
load MERGES by default: new items are inserted, existing ones refreshed, and anything the
|
||||
captures don't mention is left alone — so a partial capture of a few new items is a valid
|
||||
thing to load. Pass --replace when the captures are the whole truth and items missing from
|
||||
them should be REMOVED.
|
||||
|
||||
Target --local (default) or --remote (production; needs RECFLARE_D1 in .env).
|
||||
|
||||
Examples:
|
||||
$ runx catalog check # validate the JSON, touch nothing
|
||||
$ runx catalog load --dry-run # build the SQL, print what it would do
|
||||
$ runx catalog load # merge into the local dev database
|
||||
$ runx catalog load --remote # merge into production
|
||||
$ runx catalog load --replace # full reload: drops anything not in the captures`
|
||||
)
|
||||
@@ -0,0 +1,196 @@
|
||||
import 'zx/globals'
|
||||
|
||||
import { writeFileSync } from 'node:fs'
|
||||
import { Command } from '@commander-js/extra-typings'
|
||||
import Table from 'cli-table3'
|
||||
|
||||
import {
|
||||
buildCatalogLoad,
|
||||
CATALOG_INSERT_COLUMNS,
|
||||
CatalogKind,
|
||||
isSellableRarity,
|
||||
priceForRarity,
|
||||
subscriberPriceFor,
|
||||
} from '../../../../apps/econ/src/catalog-load'
|
||||
import { getRepoRoot } from '../path'
|
||||
import { readCaptures } from './catalog.cmd'
|
||||
|
||||
/**
|
||||
* Generate `apps/econ/static/storefronts/sf3-2025.json` — the general store as the 2025 client
|
||||
* sees it: everything sf3 already sells, PLUS every sellable row of the item catalog.
|
||||
*
|
||||
* runx storefront build
|
||||
*
|
||||
* One merged file rather than a second storefront id. The client asks for storefront 3 either
|
||||
* way; the econ worker picks WHICH file by the caller's build (`rn.ver`), so an older build
|
||||
* keeps exactly the sf3 it has always had and a newer one gets the same store with the
|
||||
* catalog's items added to it.
|
||||
*
|
||||
* A static catalog like every other `sf{N}.json`, because that is the only shape
|
||||
* `loadStorefront` reads and because browse and BUY have to agree: `findStoreItem` resolves a
|
||||
* purchase against the very same file, so an item that is not in it cannot be bought.
|
||||
*
|
||||
* The two id spaces do not collide, which is what makes the merge safe: every captured sf3 id
|
||||
* is 2764 or below (one outlier aside) and every catalog id starts at `CATALOG_ID_BASE`
|
||||
* (10000). Nothing is renumbered, and an id means the same item in both files.
|
||||
*/
|
||||
|
||||
const OUT_DIR = 'apps/econ/static/storefronts'
|
||||
|
||||
/** The captured general store, whose items the generated one is built on top of. */
|
||||
const BASE_STOREFRONT = `${OUT_DIR}/sf3.json`
|
||||
|
||||
/** The merged store the newer client is served, and the `StorefrontType` it reports. */
|
||||
const OUT_FILE = `${OUT_DIR}/sf3-2025.json`
|
||||
const STOREFRONT_TYPE = 3
|
||||
|
||||
/** RecCenterTokens — the currency the avatar storefronts sell in. */
|
||||
const CURRENCY_TYPE_TOKENS = 2
|
||||
|
||||
/**
|
||||
* Far enough out that the client never refetches — the same sentinel sf3 carries. A real
|
||||
* storefront rotates; this one is regenerated by hand, so it must not expire on its own.
|
||||
*/
|
||||
const NEXT_UPDATE = '2226-06-14T00:12:20.1324853Z'
|
||||
|
||||
const build = new Command('build')
|
||||
.description('Generate the merged 2025 general store (sf3 + the item catalog)')
|
||||
.action(async () => {
|
||||
const { avatarItems, skins } = await readCaptures()
|
||||
|
||||
// sf3's own items, carried through UNCHANGED. They keep their ids, their prices and their
|
||||
// gift-drops: the merge adds to the store the older client knows, it does not restate it.
|
||||
const basePath = path.join(getRepoRoot(), BASE_STOREFRONT)
|
||||
const base = JSON.parse(await fs.readFile(basePath, 'utf8')) as {
|
||||
StoreItems: Array<{ PurchasableItemId: number }>
|
||||
}
|
||||
|
||||
// The catalog ids come from the same loader the DB load uses, so the number in this file is
|
||||
// the number in the table. An avatar item's `item_key` IS its `AvatarItemDesc`, which is
|
||||
// what lets the two be matched up without a second numbering scheme to keep in step.
|
||||
const { rows } = buildCatalogLoad(avatarItems, skins)
|
||||
const kindAt = CATALOG_INSERT_COLUMNS.indexOf('kind')
|
||||
const catalogIdByDesc = new Map(
|
||||
rows.filter((r) => r.values[kindAt] === CatalogKind.AvatarItem).map((r) => [r.key, r.id])
|
||||
)
|
||||
|
||||
const byRarity = new Map<number, number>()
|
||||
const excluded = new Map<number, number>()
|
||||
const forSale = avatarItems.filter((item) => {
|
||||
if (isSellableRarity(item.Rarity)) return true
|
||||
excluded.set(item.Rarity, (excluded.get(item.Rarity) ?? 0) + 1)
|
||||
return false
|
||||
})
|
||||
const storeItems = forSale.map((item) => {
|
||||
const catalogId = catalogIdByDesc.get(item.AvatarItemDesc)
|
||||
if (catalogId === undefined) {
|
||||
throw new Error(`no catalog id for avatar item ${item.AvatarItemDesc}`)
|
||||
}
|
||||
const price = priceForRarity(item.Rarity)
|
||||
byRarity.set(item.Rarity, (byRarity.get(item.Rarity) ?? 0) + 1)
|
||||
const priceEntry = (p: number) => [
|
||||
{ CurrencyType: CURRENCY_TYPE_TOKENS, Price: p, StorefrontSaleData: null, Type: 0 },
|
||||
]
|
||||
return {
|
||||
// Key order and every constant field mirror sf3, because the client's parser reads
|
||||
// this shape and an sf that differs from the one known to work is a needless variable.
|
||||
GiftDrop: {
|
||||
AvatarItemDesc: item.AvatarItemDesc,
|
||||
AvatarItemType: item.AvatarItemType,
|
||||
ConsumableItemDesc: '',
|
||||
Context: 0,
|
||||
Currency: 0,
|
||||
CurrencyType: 0,
|
||||
EquipmentModificationGuid: '',
|
||||
EquipmentPrefabName: '',
|
||||
FriendlyName: item.FriendlyName,
|
||||
// sf3 has GiftDropId === PurchasableItemId on all 1161 of its items; keep that.
|
||||
GiftDropId: catalogId,
|
||||
IsQuery: false,
|
||||
ItemSetFriendlyName: '',
|
||||
ItemSetId: 0,
|
||||
Rarity: item.Rarity,
|
||||
SubscribersOnly: false,
|
||||
// The catalog's tooltip is genuinely null on some rows; the client's field is a
|
||||
// string, so this is the one place the distinction is flattened.
|
||||
Tooltip: item.Tooltip ?? '',
|
||||
Unique: true,
|
||||
},
|
||||
IsFeatured: false,
|
||||
Prices: priceEntry(price),
|
||||
// The catalog id, directly — one number, no second numbering to keep in step. It is
|
||||
// already clear of every captured storefront's ids (see `CATALOG_ID_BASE`), which is
|
||||
// what lets it be used here as-is.
|
||||
PurchasableItemId: catalogId,
|
||||
SubscriberPrices: priceEntry(subscriberPriceFor(price)),
|
||||
Type: 0,
|
||||
}
|
||||
})
|
||||
|
||||
// An id colliding across the two would make one number mean two different items depending
|
||||
// on which half answered first — refused rather than resolved by ordering, since the whole
|
||||
// point of `CATALOG_ID_BASE` is that this cannot happen.
|
||||
const baseIds = new Set(base.StoreItems.map((i) => i.PurchasableItemId))
|
||||
const collisions = storeItems.filter((i) => baseIds.has(i.PurchasableItemId))
|
||||
if (collisions.length > 0) {
|
||||
throw new Error(
|
||||
`${collisions.length} catalog id(s) collide with sf3's own, starting at ` +
|
||||
`${collisions[0]?.PurchasableItemId}. The catalog must be renumbered above them.`
|
||||
)
|
||||
}
|
||||
|
||||
const storefront = {
|
||||
NextUpdate: NEXT_UPDATE,
|
||||
StoreItems: [...base.StoreItems, ...storeItems],
|
||||
StorefrontType: STOREFRONT_TYPE,
|
||||
// Deliberately 0 — the discount is expressed ONLY in `SubscriberPrices`. Announcing it
|
||||
// again here risks a client taking the 10% off an already-discounted price and posting
|
||||
// through the server's own subscriber floor, refused as "Price has changed".
|
||||
SubscriberDiscountPercent: 0,
|
||||
}
|
||||
|
||||
writeFileSync(OUT_FILE, `${JSON.stringify(storefront, null, '\t')}\n`)
|
||||
|
||||
const table = new Table({ head: ['rarity', 'items', 'price', 'subscriber'] })
|
||||
for (const rarity of [...byRarity.keys()].sort((a, b) => a - b)) {
|
||||
const price = priceForRarity(rarity)
|
||||
table.push([
|
||||
String(rarity),
|
||||
String(byRarity.get(rarity)),
|
||||
String(price),
|
||||
String(subscriberPriceFor(price)),
|
||||
])
|
||||
}
|
||||
console.log(table.toString())
|
||||
for (const [rarity, n] of [...excluded].sort((a, b) => a[0] - b[0])) {
|
||||
console.log(chalk.yellow(`excluded ${n} item(s) of rarity ${rarity} — not for sale`))
|
||||
}
|
||||
console.log(
|
||||
chalk.green(
|
||||
`✓ wrote ${OUT_FILE}: ${storefront.StoreItems.length} items ` +
|
||||
`(${base.StoreItems.length} from sf3 + ${storeItems.length} from the catalog, ` +
|
||||
`${((await fs.stat(OUT_FILE)).size / 1024 / 1024).toFixed(1)} MB)`
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
export const storefrontCmd = new Command('storefront')
|
||||
.description('Generate the storefront catalogs the econ worker serves')
|
||||
// Bare `storefront` (no subcommand) prints help and exits cleanly, rather than commander's
|
||||
// default "missing command" error (exit 1).
|
||||
.action((_opts, command: Command) => command.outputHelp())
|
||||
.addCommand(build)
|
||||
.addHelpText(
|
||||
'after',
|
||||
`
|
||||
sf3-2025 is the general store as the 2025 client sees it: everything sf3 already sells plus
|
||||
every sellable row of the item catalog, priced by rarity. The client asks for storefront 3
|
||||
either way — the econ worker picks which file by the caller's build, so an older client keeps
|
||||
the sf3 it has always had.
|
||||
|
||||
Regenerate whenever apps/econ/static/db/*.json changes, and after every \`runx catalog load\`:
|
||||
a load renumbers catalog_id, and a stale file would list the wrong items.
|
||||
|
||||
Examples:
|
||||
$ runx storefront build # write apps/econ/static/storefronts/sf3-2025.json`
|
||||
)
|
||||
@@ -0,0 +1,122 @@
|
||||
import 'zx/globals'
|
||||
|
||||
import { getRepoRoot } from './path'
|
||||
|
||||
/**
|
||||
* Shared plumbing for talking to the one `recflare` D1 database from a CLI.
|
||||
*
|
||||
* Everything here shells out to `wrangler d1 execute` rather than calling a running worker,
|
||||
* so no deploy, no auth token and no Secrets Store read is involved — the last of which is
|
||||
* not even possible, since a Secrets Store value is write-only outside a bound Worker.
|
||||
*
|
||||
* `--local` (the default) hits the local dev database; `--remote` hits the deployed one and
|
||||
* needs the real D1 id, because the committed `wrangler.jsonc` files all carry the literal
|
||||
* placeholder `"local"` as their `database_id`. Splicing the real id into a gitignored
|
||||
* generated config is the same thing `run-wrangler-migrate` does at deploy time.
|
||||
*/
|
||||
|
||||
/** The one shared database every D1-backed worker binds. */
|
||||
export const DB_NAME = 'recflare'
|
||||
|
||||
export interface D1ExecResult {
|
||||
results: Array<Record<string, unknown>>
|
||||
success: boolean
|
||||
meta: { changes?: number; rows_read?: number }
|
||||
}
|
||||
|
||||
/** Escape a value for embedding inside a single-quoted SQL string literal. */
|
||||
export const sqlStr = (s: string): string => s.replace(/'/g, "''")
|
||||
|
||||
/** A short, loud label for which database a command is about to touch. */
|
||||
export const target = (remote: boolean): string =>
|
||||
remote ? chalk.red(`${DB_NAME} (remote)`) : chalk.cyan(`${DB_NAME} (local)`)
|
||||
|
||||
/** `--local` is the default; passing both is a mistake worth refusing rather than guessing. */
|
||||
export function resolveRemote(opts: { local?: boolean; remote?: boolean }): boolean {
|
||||
if (opts.local && opts.remote) throw new Error('pass at most one of --local / --remote')
|
||||
return opts.remote === true
|
||||
}
|
||||
|
||||
/** The deployed D1's real id, from the environment or the gitignored root .env. */
|
||||
export async function getRemoteD1Id(): Promise<string> {
|
||||
if (process.env.RECFLARE_D1) return process.env.RECFLARE_D1
|
||||
const envPath = path.join(getRepoRoot(), '.env')
|
||||
if (await fs.pathExists(envPath)) {
|
||||
const content = await fs.readFile(envPath, 'utf8')
|
||||
const m = content.match(/^\s*RECFLARE_D1\s*=\s*(.+?)\s*$/m)
|
||||
if (m) return m[1].replace(/^["']|["']$/g, '')
|
||||
}
|
||||
throw new Error('RECFLARE_D1 is not set — add the recflare D1 id to .env (see .env.example)')
|
||||
}
|
||||
|
||||
/**
|
||||
* Run wrangler `d1 execute` from one worker's directory, with the extra args a caller
|
||||
* supplies (`--command …` or `--file …`).
|
||||
*
|
||||
* The worker only decides which `wrangler.jsonc` is read and whose `node_modules` wrangler
|
||||
* resolves from — the database is the same one either way. Pick the worker that OWNS the
|
||||
* table being touched, so the config that gets spliced is the one whose migrations built it.
|
||||
*/
|
||||
async function runD1(worker: string, extraArgs: string[], remote: boolean): Promise<string> {
|
||||
const workerDir = path.join(getRepoRoot(), 'apps', worker)
|
||||
cd(workerDir)
|
||||
|
||||
const args = ['d1', 'execute', DB_NAME, ...extraArgs]
|
||||
let cleanup: (() => Promise<void>) | undefined
|
||||
|
||||
if (remote) {
|
||||
const id = await getRemoteD1Id()
|
||||
const src = await fs.readFile(path.join(workerDir, 'wrangler.jsonc'), 'utf8')
|
||||
const generated = src.replace(/("database_id"\s*:\s*")[^"]*(")/, `$1${id}$2`)
|
||||
const genPath = path.join(workerDir, 'wrangler.generated.jsonc')
|
||||
await fs.writeFile(genPath, generated)
|
||||
cleanup = () => fs.remove(genPath)
|
||||
// `--yes` because wrangler asks for confirmation before touching the deployed database
|
||||
// and a swallowed prompt looks exactly like a hang.
|
||||
args.push('--config', 'wrangler.generated.jsonc', '--remote', '--yes')
|
||||
} else {
|
||||
args.push('--local')
|
||||
}
|
||||
|
||||
try {
|
||||
// Via `pnpm exec` so wrangler resolves from the worker's node_modules (it isn't a
|
||||
// dependency of @repo/tools, so it's not on this process's PATH).
|
||||
const out = await $`pnpm exec wrangler ${args}`.quiet()
|
||||
return out.stdout
|
||||
} finally {
|
||||
if (cleanup) await cleanup()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one SQL statement and parse the result.
|
||||
*
|
||||
* Read the RESULTS to find out what happened, not the meta: wrangler's `--json` meta carries
|
||||
* only a duration, with no reliable `changes` count, so a statement that needs to know what it
|
||||
* matched has to say `RETURNING`.
|
||||
*/
|
||||
export async function execSql(
|
||||
sql: string,
|
||||
remote: boolean,
|
||||
worker = 'auth'
|
||||
): Promise<D1ExecResult> {
|
||||
const stdout = await runD1(worker, ['--command', sql, '--json'], remote)
|
||||
// wrangler --json prints a one-element array of results to stdout.
|
||||
const start = stdout.indexOf('[')
|
||||
if (start === -1) throw new Error(`unexpected d1 execute output:\n${stdout}`)
|
||||
const parsed = JSON.parse(stdout.slice(start)) as D1ExecResult[]
|
||||
const first = parsed[0]
|
||||
if (!first) throw new Error(`empty d1 execute result:\n${stdout}`)
|
||||
return first
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingest a whole `.sql` file. Used for bulk loads far too large for `--command`, where
|
||||
* wrangler splits the file into statements and batches them itself.
|
||||
*
|
||||
* Not `--json`: a multi-thousand-statement file answers with a result object per statement,
|
||||
* which is megabytes of nothing. The human-readable output is returned for the caller.
|
||||
*/
|
||||
export async function execSqlFile(file: string, remote: boolean, worker = 'auth'): Promise<string> {
|
||||
return await runD1(worker, ['--file', file], remote)
|
||||
}
|
||||
Reference in New Issue
Block a user