TheSunIsLeaking/recflare
3
0
Fork 0
mirror of https://github.com/djdevin/recflare.git synced 2026-09-08 14:41:28 -07:00
Code Issues Packages Projects Releases Wiki Activity

[packages] tools for catalog building

Browse Source
This commit is contained in:
Devin Zuczek
2026-08-27 22:43:41 -04:00
parent a169cf3e6e
commit f8ca01dd4d
5 changed files with 680 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/tools/src/bin/runx.cmd.ts
+4
View File
@@ -5,11 +5,13 @@ import { catchProcessError } from '@jahands/cli-tools/proc'
import { adminCmd } from '../cmd/admin.cmd'
import { buildCmd } from '../cmd/build.cmd'
import { catalogCmd } from '../cmd/catalog.cmd'
import { checkCmd } from '../cmd/check.cmd'
import { ciCmd } from '../cmd/ci.cmd'
import { devCmd } from '../cmd/dev.cmd'
import { fixCmd } from '../cmd/fix.cmd'
import { shfmtCmd } from '../cmd/shfmt.cmd'
import { storefrontCmd } from '../cmd/storefront.cmd'
import { updateCmd } from '../cmd/update.cmd'
program
@@ -20,6 +22,8 @@ program
// a typescript CLI is nicer for more complex things.
.addCommand(adminCmd)
.addCommand(catalogCmd)
.addCommand(storefrontCmd)
.addCommand(fixCmd)
.addCommand(buildCmd)
.addCommand(checkCmd)
packages/tools/src/cmd/admin.cmd.ts
+11 -79
View File
@@ -1,11 +1,12 @@
import * as readline from 'node:readline'
import { Command } from '@commander-js/extra-typings'
import Table from 'cli-table3'
import { getRepoRoot } from '../path'
import { execSql, resolveRemote, sqlStr, target } from '../d1'
import { hashPassword } from '../password'
import type { D1ExecResult } from '../d1'
/**
* Operator-facing admin tools for the shared `recflare` D1 database. Each command
* shells out to `wrangler d1 execute recflare` — no running worker or auth token
@@ -19,18 +20,6 @@ import { hashPassword } from '../password'
* runx admin grant-developer --account 1 [--revoke] [--remote]
*/
/** The one shared database every D1-backed worker binds. */
const DB_NAME = 'recflare'
interface D1ExecResult {
results: Array<Record<string, unknown>>
success: boolean
meta: { changes?: number; rows_read?: number }
}
/** Escape a value for embedding inside a single-quoted SQL string literal. */
const sqlStr = (s: string): string => s.replace(/'/g, "''")
/**
* Resolve the account selector into a SQL WHERE fragment. Exactly one of
* `--account` / `--username` must be given. Account ids are validated numeric;
@@ -50,59 +39,6 @@ function whereClause(account?: string, username?: string): { where: string; labe
}
}
/** The deployed D1's real id, from the environment or the gitignored root .env. */
async function getRemoteD1Id(): Promise<string> {
if (process.env.RECFLARE_D1) return process.env.RECFLARE_D1
const envPath = path.join(getRepoRoot(), '.env')
if (await fs.pathExists(envPath)) {
const content = await fs.readFile(envPath, 'utf8')
const m = content.match(/^\s*RECFLARE_D1\s*=\s*(.+?)\s*$/m)
if (m) return m[1].replace(/^["']|["']$/g, '')
}
throw new Error('RECFLARE_D1 is not set — add the recflare D1 id to .env (see .env.example)')
}
/**
* Run a SQL statement against the shared database via wrangler. Runs from the auth
* worker's directory (it owns the accounts schema and binds the DB). For `--remote`
* the committed wrangler.jsonc's "local" database_id placeholder is spliced with the
* real id into a gitignored generated config — exactly like run-wrangler-migrate.
*/
async function execSql(sql: string, remote: boolean): Promise<D1ExecResult> {
const authDir = path.join(getRepoRoot(), 'apps', 'auth')
cd(authDir)
const args = ['d1', 'execute', DB_NAME, '--command', sql, '--json']
let cleanup: (() => Promise<void>) | undefined
if (remote) {
const id = await getRemoteD1Id()
const src = await fs.readFile(path.join(authDir, 'wrangler.jsonc'), 'utf8')
const generated = src.replace(/("database_id"\s*:\s*")[^"]*(")/, `$1${id}$2`)
const genPath = path.join(authDir, 'wrangler.generated.jsonc')
await fs.writeFile(genPath, generated)
cleanup = () => fs.remove(genPath)
args.push('--config', 'wrangler.generated.jsonc', '--remote')
} else {
args.push('--local')
}
try {
// Via `pnpm exec` so wrangler resolves from the auth worker's node_modules
// (it isn't a dependency of @repo/tools, so it's not on this process's PATH).
const out = await $`pnpm exec wrangler ${args}`.quiet()
// wrangler --json prints a one-element array of results to stdout.
const start = out.stdout.indexOf('[')
if (start === -1) throw new Error(`unexpected d1 execute output:\n${out.stdout}`)
const parsed = JSON.parse(out.stdout.slice(start)) as D1ExecResult[]
const first = parsed[0]
if (!first) throw new Error(`empty d1 execute result:\n${out.stdout}`)
return first
} finally {
if (cleanup) await cleanup()
}
}
/** Prompt for a line of input without echoing what's typed (for passwords). */
function promptHidden(query: string): Promise<string> {
return new Promise((resolve) => {
@@ -131,7 +67,9 @@ async function resolvePassword(flag?: string): Promise<string> {
if (!process.stdin.isTTY) {
const chunks: Buffer[] = []
for await (const chunk of process.stdin) chunks.push(chunk as Buffer)
const piped = Buffer.concat(chunks).toString('utf8').replace(/\r?\n$/, '')
const piped = Buffer.concat(chunks)
.toString('utf8')
.replace(/\r?\n$/, '')
if (piped === '') throw new Error('no password provided on stdin')
return piped
}
@@ -142,16 +80,6 @@ async function resolvePassword(flag?: string): Promise<string> {
return first
}
/** A short, loud label for which database a command is about to touch. */
const target = (remote: boolean): string =>
remote ? chalk.red(`${DB_NAME} (remote)`) : chalk.cyan(`${DB_NAME} (local)`)
/** Resolve the --local/--remote target flags. Local is the default. */
function resolveRemote(opts: { local?: boolean; remote?: boolean }): boolean {
if (opts.local && opts.remote) throw new Error('pass at most one of --local / --remote')
return opts.remote === true
}
/**
* Fail when a WHERE-scoped UPDATE matched no row (i.e. no such account). Relies on
* the statement's `RETURNING account_id` — wrangler's `--json` meta doesn't reliably
@@ -251,7 +179,11 @@ const lookup = new Command('lookup')
return
}
const asText = (v: unknown): string =>
v == null ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v as number | string | boolean)
v == null
? ''
: typeof v === 'object'
? JSON.stringify(v)
: String(v as number | string | boolean)
const boolKeys = new Set(['hasPassword', 'isDeveloper', 'isModerator'])
const table = new Table()
for (const [key, value] of Object.entries(row)) {
packages/tools/src/cmd/catalog.cmd.ts
+347
View File
@@ -0,0 +1,347 @@
import { Command } from '@commander-js/extra-typings'
import Table from 'cli-table3'
import {
buildCatalogLoad,
CATALOG_ID_BASE,
CATALOG_INSERT_COLUMNS,
} from '../../../../apps/econ/src/catalog-load'
import { execSql, execSqlFile, resolveRemote, target } from '../d1'
import { getRepoRoot } from '../path'
import type {
AvatarItemCapture,
CatalogCollision,
CatalogLoadRow,
CatalogValue,
SkinCapture,
} from '../../../../apps/econ/src/catalog-load'
/**
* Load the item catalog into the shared `recflare` D1 database.
*
* The `catalog` table's STRUCTURE is a migration (apps/econ/migrations/0015_catalog.sql); its
* CONTENTS are not. The game's item list changes without the schema changing, so shipping the
* rows as a migration would mean a migration and a deploy per refresh, an ever-growing pile of
* near-identical data migrations, and no way to reload without writing another one. This
* command is the reload.
*
* runx catalog load [--remote] [--replace] [--dry-run]
* runx catalog check
*
* The mapping from capture to row lives in the econ worker's `catalog-load.ts`, beside the
* schema it fills, so a column added there and a column added to the loader cannot drift. That
* module deliberately touches no Workers types, which is what lets this Node CLI import it.
*/
/** The worker that owns the table — whose wrangler.jsonc and node_modules wrangler uses. */
const OWNER = 'econ'
const AVATAR_ITEMS = 'apps/econ/static/db/avatar-items.json'
const SKINS = 'apps/econ/static/db/skins.json'
/**
* Rows per INSERT. Batched because thousands of single-row statements parse far more slowly
* than a few dozen multi-row ones, and D1 charges per statement on the remote path.
*/
const CHUNK = 100
/**
* One SQL literal. Booleans become 1/0 (SQLite has no boolean), `undefined` is treated as the
* NULL it stands for — a key the capture omitted rather than set — and quotes are doubled.
* Item names are the only free text here, but escaping is what makes running this against a
* fresh capture safe rather than lucky.
*/
function lit(v: CatalogValue): string {
if (v === null || v === undefined) return 'NULL'
if (typeof v === 'boolean') return v ? '1' : '0'
if (typeof v === 'number') return String(v)
return `'${v.replaceAll("'", "''")}'`
}
/** Read one capture, tolerating the BOM the exports carry (`JSON.parse` rejects U+FEFF). */
async function readCapture<T>(relPath: string): Promise<T[]> {
const full = path.join(getRepoRoot(), relPath)
if (!(await fs.pathExists(full))) throw new Error(`no capture at ${relPath}`)
const text = (await fs.readFile(full, 'utf8')).replace(/^/, '')
const parsed = JSON.parse(text) as unknown
if (!Array.isArray(parsed)) throw new Error(`${relPath} is not a JSON array`)
return parsed as T[]
}
/** Both captures, as the loader and the storefront generator read them. */
export async function readCaptures(): Promise<{
avatarItems: AvatarItemCapture[]
skins: SkinCapture[]
}> {
const [avatarItems, skins] = await Promise.all([
readCapture<AvatarItemCapture>(AVATAR_ITEMS),
readCapture<SkinCapture>(SKINS),
])
return { avatarItems, skins }
}
/** Print the duplicate keys a load dropped. Never silent: that is the whole point of them. */
function reportCollisions(collisions: CatalogCollision[]): void {
if (collisions.length === 0) return
console.warn(
chalk.yellow(
`\n${collisions.length} duplicate item_key(s) in the captures — first occurrence kept:`
)
)
const table = new Table({ head: ['item_key', 'kept', 'dropped'] })
for (const c of collisions) table.push([c.key, c.kept, c.dropped])
console.warn(table.toString())
console.warn(
chalk.yellow('A repeat is a defect in the capture — fix the source JSON and load again.\n')
)
}
/**
* How many rows the catalog holds, per kind. Read before and after a load: wrangler's meta
* carries no usable row count, so counting the table is the only honest way to say what a load
* actually did.
*/
async function countRows(remote: boolean): Promise<Record<string, number>> {
const res = await execSql('SELECT kind, COUNT(*) AS n FROM catalog GROUP BY kind', remote, OWNER)
return Object.fromEntries(res.results.map((r) => [String(r.kind), Number(r.n)]))
}
const total = (counts: Record<string, number>): number =>
Object.values(counts).reduce((a, b) => a + b, 0)
/**
* The `DO UPDATE` half of the upsert: every column except the conflict target, taken from the
* row that was being inserted. Derived from {@link CATALOG_INSERT_COLUMNS} rather than written
* out, so a column added to the table is carried by a merge automatically — a hand-maintained
* list would quietly stop updating whatever was forgotten.
*/
const CONFLICT_UPDATE = CATALOG_INSERT_COLUMNS.filter((c) => c !== 'item_key')
.map((c) => `${c} = excluded.${c}`)
.join(', ')
/**
* Prove the load actually landed, and refuse to report success otherwise.
*
* This exists because a load once reported "✓ catalog loaded" over a database it had written
* nothing to. Counting rows back is not paranoia: wrangler's meta carries no usable row count,
* the remote and local paths differ enough that one can break while the other works, and a
* `--file` that fails partway leaves a partial catalog rather than an error. The command must
* either see its own rows in the table or say it failed.
*/
async function verifyLoad(
rows: CatalogLoadRow[],
after: Record<string, number>,
replace: boolean,
remote: boolean
): Promise<void> {
const got = total(after)
// A replace empties first, so the count is exact. A merge only adds, so the table must hold at
// least what was just written — more is fine, that is the rows the captures did not mention.
if (replace ? got !== rows.length : got < rows.length) {
throw new Error(
`load did not land: expected ${replace ? '' : 'at least '}${rows.length} rows, ` +
`the table holds ${got}. Nothing was verified as written — re-run it.`
)
}
// Counts alone can be satisfied by the rows that were already there, so spot-check actual keys
// from across the file. The last one matters most: a load cut short by a failed batch loses
// the tail while the count still looks plausible.
const probes = [rows[0], rows[Math.floor(rows.length / 2)], rows[rows.length - 1]].filter(
(r): r is CatalogLoadRow => r !== undefined
)
const list = probes.map((r) => `'${r.key.replaceAll("'", "''")}'`).join(', ')
const res = await execSql(
`SELECT COUNT(*) AS n FROM catalog WHERE item_key IN (${list})`,
remote,
OWNER
)
const found = Number(res.results[0]?.n ?? 0)
if (found !== probes.length) {
throw new Error(
`load did not land: ${probes.length - found} of ${probes.length} probe rows are missing ` +
`from the table. The catalog may be partially written — re-run it.`
)
}
// Every row must have come out with a numeric handle. A NULL here is a load that stopped
// between writing rows and numbering them, which nothing else would notice until a lookup by
// number quietly returned nothing.
const unnumbered = await execSql(
'SELECT COUNT(*) AS n FROM catalog WHERE catalog_id IS NULL',
remote,
OWNER
)
const missing = Number(unnumbered.results[0]?.n ?? 0)
if (missing > 0) {
throw new Error(`load did not finish: ${missing} row(s) have no catalog_id. Re-run it.`)
}
}
const load = new Command('load')
.description('Load the item catalog from the captured JSON (merges by default)')
.option('--local', 'Target the local dev database (the default).', false)
.option('--remote', 'Target the deployed database instead of the local dev database.', false)
.option(
'--replace',
'Empty the catalog first, so rows absent from the captures are REMOVED.',
false
)
.option('--dry-run', 'Build and validate the SQL, print what it would do, change nothing.', false)
.action(async (opts) => {
const remote = resolveRemote(opts)
const { avatarItems, skins } = await readCaptures()
const { rows, collisions } = buildCatalogLoad(avatarItems, skins)
console.log(
`${opts.replace ? 'Replacing the catalog with' : 'Merging'} ${rows.length} rows ` +
`(${avatarItems.length} avatar items, ${skins.length} skins) into ${target(remote)}`
)
reportCollisions(collisions)
// MERGE is the default: insert what is new, refresh what already exists, and leave
// anything the captures do not mention alone. That is what makes a PARTIAL capture useful
// — a handful of newly-datamined items can be loaded without having to re-export the whole
// catalog first, and without a partial file silently wiping the rest.
//
// The cost is that a merge can never REMOVE an item: something dropped from the captures
// stays in the table forever. `--replace` is the authoritative reload for when the
// captures are meant to be the whole truth, and it is opt-in because it is the destructive
// one — pointed at a partial capture it would delete everything the file omits.
//
// NO `BEGIN TRANSACTION` / `COMMIT`. Remote D1 REFUSES them outright ("To execute a
// transaction, please use the state.storage.transaction() ... APIs instead of the SQL
// BEGIN TRANSACTION or SAVEPOINT statements"), so a file carrying them loads nothing at
// all against production while working fine locally. Atomicity comes from D1 running each
// batch of statements in its own implicit transaction; across batches there is none, so a
// load that dies midway can leave the catalog partial. That is what {@link verifyLoad}
// below is for, and why re-running is always safe: a merge is idempotent, and a replace
// starts by emptying the table.
const statements: string[] = []
if (opts.replace) {
statements.push('DELETE FROM catalog;')
} else {
// Clear every existing number BEFORE assigning any. `catalog_id` is unique, and this
// load is about to hand out 1..N: without this, a row already holding one of those
// numbers (because it was in an earlier load and this capture no longer mentions it)
// collides and the whole merge fails. Nulling first also means a row left un-numbered
// afterwards is visibly a row the captures did not mention, which the tail statement
// below then numbers above N.
statements.push('UPDATE catalog SET catalog_id = NULL;')
}
for (let start = 0; start < rows.length; start += CHUNK) {
const batch = rows.slice(start, start + CHUNK)
statements.push(
`INSERT INTO catalog (${CATALOG_INSERT_COLUMNS.join(', ')}) VALUES\n` +
batch.map((r) => `\t(${r.values.map(lit).join(', ')})`).join(',\n') +
// Harmless after a DELETE (nothing can conflict), and kept there anyway so both
// paths run the identical statement rather than two shapes that could diverge.
`\nON CONFLICT(item_key) DO UPDATE SET ${CONFLICT_UPDATE};`
)
}
if (!opts.replace) {
// Anything still un-numbered is a row the captures did not mention — a merge keeps those,
// so they need handles too. They go ABOVE the highest id this load assigned, so they can
// never collide with it, ordered by rowid for determinism. Usually this matches nothing
// at all, and it is a single statement either way.
const highest = CATALOG_ID_BASE + rows.length - 1
statements.push(
`UPDATE catalog SET catalog_id = ${highest} +\n` +
`\t(SELECT COUNT(*) FROM catalog c WHERE c.catalog_id IS NULL AND c.rowid <= catalog.rowid)\n` +
`WHERE catalog_id IS NULL;`
)
}
const sql = statements.join('\n')
if (opts.dryRun) {
console.log(
chalk.cyan(
`--dry-run: built ${statements.length} statements (${(sql.length / 1024).toFixed(0)} KB); nothing was written.`
)
)
return
}
const before = await countRows(remote)
// Written to a temp file rather than passed as `--command`: a multi-megabyte argv is not
// something to rely on, and `--file` is the path wrangler batches for us.
const file = path.join(os.tmpdir(), `recflare-catalog-${Date.now()}.sql`)
await fs.writeFile(file, sql)
try {
await execSqlFile(file, remote, OWNER)
} finally {
await fs.remove(file)
}
const after = await countRows(remote)
const table = new Table({ head: ['kind', 'before', 'after'] })
for (const kind of new Set([...Object.keys(before), ...Object.keys(after)])) {
table.push([kind, String(before[kind] ?? 0), String(after[kind] ?? 0)])
}
console.log(table.toString())
await verifyLoad(rows, after, opts.replace, remote)
// On a merge the table only grows, so the growth IS the number of new items and the rest of
// what was written updated a row that already existed. Worth saying out loud: "3370 rows
// loaded" reads like 3370 changes when it may well have been six.
if (!opts.replace) {
const added = total(after) - total(before)
console.log(`${added} new, ${rows.length - added} updated in place, 0 removed`)
}
console.log(chalk.green(`✓ catalog loaded into ${target(remote)}`))
})
const check = new Command('check')
.description('Validate the captured JSON without touching any database')
.action(async () => {
const { avatarItems, skins } = await readCaptures()
const { rows, collisions } = buildCatalogLoad(avatarItems, skins)
const table = new Table()
table.push(
{ 'avatar items': String(avatarItems.length) },
{ skins: String(skins.length) },
{ 'rows to load': String(rows.length) },
{ 'duplicate keys': String(collisions.length) }
)
console.log(table.toString())
reportCollisions(collisions)
// An empty capture is far more likely to be a broken export than a real one, and loading it
// with --replace would swap a good catalog for nothing.
if (rows.length === 0)
throw new Error('the captures produced no rows — refusing to call that ok')
console.log(chalk.green('✓ captures are loadable'))
})
export const catalogCmd = new Command('catalog')
.description('Load the item catalog into the shared recflare D1 database')
// Bare `catalog` (no subcommand) prints help and exits cleanly, rather than commander's
// default "missing command" error (exit 1).
.action((_opts, command: Command) => command.outputHelp())
.addCommand(load)
.addCommand(check)
.addHelpText(
'after',
`
The catalog's structure is a migration; its contents are not — reload them here whenever
apps/econ/static/db/*.json changes, with no migration and no deploy.
load MERGES by default: new items are inserted, existing ones refreshed, and anything the
captures don't mention is left alone — so a partial capture of a few new items is a valid
thing to load. Pass --replace when the captures are the whole truth and items missing from
them should be REMOVED.
Target --local (default) or --remote (production; needs RECFLARE_D1 in .env).
Examples:
$ runx catalog check # validate the JSON, touch nothing
$ runx catalog load --dry-run # build the SQL, print what it would do
$ runx catalog load # merge into the local dev database
$ runx catalog load --remote # merge into production
$ runx catalog load --replace # full reload: drops anything not in the captures`
)
packages/tools/src/cmd/storefront.cmd.ts
+196
View File
@@ -0,0 +1,196 @@
import 'zx/globals'
import { writeFileSync } from 'node:fs'
import { Command } from '@commander-js/extra-typings'
import Table from 'cli-table3'
import {
buildCatalogLoad,
CATALOG_INSERT_COLUMNS,
CatalogKind,
isSellableRarity,
priceForRarity,
subscriberPriceFor,
} from '../../../../apps/econ/src/catalog-load'
import { getRepoRoot } from '../path'
import { readCaptures } from './catalog.cmd'
/**
* Generate `apps/econ/static/storefronts/sf3-2025.json` — the general store as the 2025 client
* sees it: everything sf3 already sells, PLUS every sellable row of the item catalog.
*
* runx storefront build
*
* One merged file rather than a second storefront id. The client asks for storefront 3 either
* way; the econ worker picks WHICH file by the caller's build (`rn.ver`), so an older build
* keeps exactly the sf3 it has always had and a newer one gets the same store with the
* catalog's items added to it.
*
* A static catalog like every other `sf{N}.json`, because that is the only shape
* `loadStorefront` reads and because browse and BUY have to agree: `findStoreItem` resolves a
* purchase against the very same file, so an item that is not in it cannot be bought.
*
* The two id spaces do not collide, which is what makes the merge safe: every captured sf3 id
* is 2764 or below (one outlier aside) and every catalog id starts at `CATALOG_ID_BASE`
* (10000). Nothing is renumbered, and an id means the same item in both files.
*/
const OUT_DIR = 'apps/econ/static/storefronts'
/** The captured general store, whose items the generated one is built on top of. */
const BASE_STOREFRONT = `${OUT_DIR}/sf3.json`
/** The merged store the newer client is served, and the `StorefrontType` it reports. */
const OUT_FILE = `${OUT_DIR}/sf3-2025.json`
const STOREFRONT_TYPE = 3
/** RecCenterTokens — the currency the avatar storefronts sell in. */
const CURRENCY_TYPE_TOKENS = 2
/**
* Far enough out that the client never refetches — the same sentinel sf3 carries. A real
* storefront rotates; this one is regenerated by hand, so it must not expire on its own.
*/
const NEXT_UPDATE = '2226-06-14T00:12:20.1324853Z'
const build = new Command('build')
.description('Generate the merged 2025 general store (sf3 + the item catalog)')
.action(async () => {
const { avatarItems, skins } = await readCaptures()
// sf3's own items, carried through UNCHANGED. They keep their ids, their prices and their
// gift-drops: the merge adds to the store the older client knows, it does not restate it.
const basePath = path.join(getRepoRoot(), BASE_STOREFRONT)
const base = JSON.parse(await fs.readFile(basePath, 'utf8')) as {
StoreItems: Array<{ PurchasableItemId: number }>
}
// The catalog ids come from the same loader the DB load uses, so the number in this file is
// the number in the table. An avatar item's `item_key` IS its `AvatarItemDesc`, which is
// what lets the two be matched up without a second numbering scheme to keep in step.
const { rows } = buildCatalogLoad(avatarItems, skins)
const kindAt = CATALOG_INSERT_COLUMNS.indexOf('kind')
const catalogIdByDesc = new Map(
rows.filter((r) => r.values[kindAt] === CatalogKind.AvatarItem).map((r) => [r.key, r.id])
)
const byRarity = new Map<number, number>()
const excluded = new Map<number, number>()
const forSale = avatarItems.filter((item) => {
if (isSellableRarity(item.Rarity)) return true
excluded.set(item.Rarity, (excluded.get(item.Rarity) ?? 0) + 1)
return false
})
const storeItems = forSale.map((item) => {
const catalogId = catalogIdByDesc.get(item.AvatarItemDesc)
if (catalogId === undefined) {
throw new Error(`no catalog id for avatar item ${item.AvatarItemDesc}`)
}
const price = priceForRarity(item.Rarity)
byRarity.set(item.Rarity, (byRarity.get(item.Rarity) ?? 0) + 1)
const priceEntry = (p: number) => [
{ CurrencyType: CURRENCY_TYPE_TOKENS, Price: p, StorefrontSaleData: null, Type: 0 },
]
return {
// Key order and every constant field mirror sf3, because the client's parser reads
// this shape and an sf that differs from the one known to work is a needless variable.
GiftDrop: {
AvatarItemDesc: item.AvatarItemDesc,
AvatarItemType: item.AvatarItemType,
ConsumableItemDesc: '',
Context: 0,
Currency: 0,
CurrencyType: 0,
EquipmentModificationGuid: '',
EquipmentPrefabName: '',
FriendlyName: item.FriendlyName,
// sf3 has GiftDropId === PurchasableItemId on all 1161 of its items; keep that.
GiftDropId: catalogId,
IsQuery: false,
ItemSetFriendlyName: '',
ItemSetId: 0,
Rarity: item.Rarity,
SubscribersOnly: false,
// The catalog's tooltip is genuinely null on some rows; the client's field is a
// string, so this is the one place the distinction is flattened.
Tooltip: item.Tooltip ?? '',
Unique: true,
},
IsFeatured: false,
Prices: priceEntry(price),
// The catalog id, directly — one number, no second numbering to keep in step. It is
// already clear of every captured storefront's ids (see `CATALOG_ID_BASE`), which is
// what lets it be used here as-is.
PurchasableItemId: catalogId,
SubscriberPrices: priceEntry(subscriberPriceFor(price)),
Type: 0,
}
})
// An id colliding across the two would make one number mean two different items depending
// on which half answered first — refused rather than resolved by ordering, since the whole
// point of `CATALOG_ID_BASE` is that this cannot happen.
const baseIds = new Set(base.StoreItems.map((i) => i.PurchasableItemId))
const collisions = storeItems.filter((i) => baseIds.has(i.PurchasableItemId))
if (collisions.length > 0) {
throw new Error(
`${collisions.length} catalog id(s) collide with sf3's own, starting at ` +
`${collisions[0]?.PurchasableItemId}. The catalog must be renumbered above them.`
)
}
const storefront = {
NextUpdate: NEXT_UPDATE,
StoreItems: [...base.StoreItems, ...storeItems],
StorefrontType: STOREFRONT_TYPE,
// Deliberately 0 — the discount is expressed ONLY in `SubscriberPrices`. Announcing it
// again here risks a client taking the 10% off an already-discounted price and posting
// through the server's own subscriber floor, refused as "Price has changed".
SubscriberDiscountPercent: 0,
}
writeFileSync(OUT_FILE, `${JSON.stringify(storefront, null, '\t')}\n`)
const table = new Table({ head: ['rarity', 'items', 'price', 'subscriber'] })
for (const rarity of [...byRarity.keys()].sort((a, b) => a - b)) {
const price = priceForRarity(rarity)
table.push([
String(rarity),
String(byRarity.get(rarity)),
String(price),
String(subscriberPriceFor(price)),
])
}
console.log(table.toString())
for (const [rarity, n] of [...excluded].sort((a, b) => a[0] - b[0])) {
console.log(chalk.yellow(`excluded ${n} item(s) of rarity ${rarity} — not for sale`))
}
console.log(
chalk.green(
`✓ wrote ${OUT_FILE}: ${storefront.StoreItems.length} items ` +
`(${base.StoreItems.length} from sf3 + ${storeItems.length} from the catalog, ` +
`${((await fs.stat(OUT_FILE)).size / 1024 / 1024).toFixed(1)} MB)`
)
)
})
export const storefrontCmd = new Command('storefront')
.description('Generate the storefront catalogs the econ worker serves')
// Bare `storefront` (no subcommand) prints help and exits cleanly, rather than commander's
// default "missing command" error (exit 1).
.action((_opts, command: Command) => command.outputHelp())
.addCommand(build)
.addHelpText(
'after',
`
sf3-2025 is the general store as the 2025 client sees it: everything sf3 already sells plus
every sellable row of the item catalog, priced by rarity. The client asks for storefront 3
either way — the econ worker picks which file by the caller's build, so an older client keeps
the sf3 it has always had.
Regenerate whenever apps/econ/static/db/*.json changes, and after every \`runx catalog load\`:
a load renumbers catalog_id, and a stale file would list the wrong items.
Examples:
$ runx storefront build # write apps/econ/static/storefronts/sf3-2025.json`
)
packages/tools/src/d1.ts
+122
View File
@@ -0,0 +1,122 @@
import 'zx/globals'
import { getRepoRoot } from './path'
/**
* Shared plumbing for talking to the one `recflare` D1 database from a CLI.
*
* Everything here shells out to `wrangler d1 execute` rather than calling a running worker,
* so no deploy, no auth token and no Secrets Store read is involved — the last of which is
* not even possible, since a Secrets Store value is write-only outside a bound Worker.
*
* `--local` (the default) hits the local dev database; `--remote` hits the deployed one and
* needs the real D1 id, because the committed `wrangler.jsonc` files all carry the literal
* placeholder `"local"` as their `database_id`. Splicing the real id into a gitignored
* generated config is the same thing `run-wrangler-migrate` does at deploy time.
*/
/** The one shared database every D1-backed worker binds. */
export const DB_NAME = 'recflare'
export interface D1ExecResult {
results: Array<Record<string, unknown>>
success: boolean
meta: { changes?: number; rows_read?: number }
}
/** Escape a value for embedding inside a single-quoted SQL string literal. */
export const sqlStr = (s: string): string => s.replace(/'/g, "''")
/** A short, loud label for which database a command is about to touch. */
export const target = (remote: boolean): string =>
remote ? chalk.red(`${DB_NAME} (remote)`) : chalk.cyan(`${DB_NAME} (local)`)
/** `--local` is the default; passing both is a mistake worth refusing rather than guessing. */
export function resolveRemote(opts: { local?: boolean; remote?: boolean }): boolean {
if (opts.local && opts.remote) throw new Error('pass at most one of --local / --remote')
return opts.remote === true
}
/** The deployed D1's real id, from the environment or the gitignored root .env. */
export async function getRemoteD1Id(): Promise<string> {
if (process.env.RECFLARE_D1) return process.env.RECFLARE_D1
const envPath = path.join(getRepoRoot(), '.env')
if (await fs.pathExists(envPath)) {
const content = await fs.readFile(envPath, 'utf8')
const m = content.match(/^\s*RECFLARE_D1\s*=\s*(.+?)\s*$/m)
if (m) return m[1].replace(/^["']|["']$/g, '')
}
throw new Error('RECFLARE_D1 is not set — add the recflare D1 id to .env (see .env.example)')
}
/**
* Run wrangler `d1 execute` from one worker's directory, with the extra args a caller
* supplies (`--command …` or `--file …`).
*
* The worker only decides which `wrangler.jsonc` is read and whose `node_modules` wrangler
* resolves from — the database is the same one either way. Pick the worker that OWNS the
* table being touched, so the config that gets spliced is the one whose migrations built it.
*/
async function runD1(worker: string, extraArgs: string[], remote: boolean): Promise<string> {
const workerDir = path.join(getRepoRoot(), 'apps', worker)
cd(workerDir)
const args = ['d1', 'execute', DB_NAME, ...extraArgs]
let cleanup: (() => Promise<void>) | undefined
if (remote) {
const id = await getRemoteD1Id()
const src = await fs.readFile(path.join(workerDir, 'wrangler.jsonc'), 'utf8')
const generated = src.replace(/("database_id"\s*:\s*")[^"]*(")/, `$1${id}$2`)
const genPath = path.join(workerDir, 'wrangler.generated.jsonc')
await fs.writeFile(genPath, generated)
cleanup = () => fs.remove(genPath)
// `--yes` because wrangler asks for confirmation before touching the deployed database
// and a swallowed prompt looks exactly like a hang.
args.push('--config', 'wrangler.generated.jsonc', '--remote', '--yes')
} else {
args.push('--local')
}
try {
// Via `pnpm exec` so wrangler resolves from the worker's node_modules (it isn't a
// dependency of @repo/tools, so it's not on this process's PATH).
const out = await $`pnpm exec wrangler ${args}`.quiet()
return out.stdout
} finally {
if (cleanup) await cleanup()
}
}
/**
* Run one SQL statement and parse the result.
*
* Read the RESULTS to find out what happened, not the meta: wrangler's `--json` meta carries
* only a duration, with no reliable `changes` count, so a statement that needs to know what it
* matched has to say `RETURNING`.
*/
export async function execSql(
sql: string,
remote: boolean,
worker = 'auth'
): Promise<D1ExecResult> {
const stdout = await runD1(worker, ['--command', sql, '--json'], remote)
// wrangler --json prints a one-element array of results to stdout.
const start = stdout.indexOf('[')
if (start === -1) throw new Error(`unexpected d1 execute output:\n${stdout}`)
const parsed = JSON.parse(stdout.slice(start)) as D1ExecResult[]
const first = parsed[0]
if (!first) throw new Error(`empty d1 execute result:\n${stdout}`)
return first
}
/**
* Ingest a whole `.sql` file. Used for bulk loads far too large for `--command`, where
* wrangler splits the file into statements and batches them itself.
*
* Not `--json`: a multi-thousand-statement file answers with a result object per statement,
* which is megabytes of nothing. The human-readable output is returned for the caller.
*/
export async function execSqlFile(file: string, remote: boolean, worker = 'auth'): Promise<string> {
return await runD1(worker, ['--file', file], remote)
}