mirror of
https://github.com/djdevin/recflare.git
synced 2026-10-25 23:18:51 -07:00
[api] catalog loader
This commit is contained in:
Devin Zuczek
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Turns an export of first-party custom avatar items into a D1 migration that seeds the
|
||||
* `custom_avatar_item` table with them, one `INSERT` per record, the way
|
||||
* `apps/rooms/migrations/0002_import_rooms.sql` seeds the rooms.
|
||||
* `custom_avatar_item` table with them, the way `apps/rooms/migrations/0002_import_rooms.sql`
|
||||
* seeds the rooms.
|
||||
*
|
||||
* bun apps/api/scripts/import-custom-avatar-items.ts <export.json> <NNNN>
|
||||
*
|
||||
@@ -9,11 +9,11 @@
|
||||
* them (a `{ Results: [...] }` page or a single record are accepted too). `NNNN` is the
|
||||
* migration number; the file is written to `apps/api/migrations/NNNN_import_custom_avatar_items.sql`.
|
||||
*
|
||||
* Each record is stored VERBATIM as the row's JSON (the table is a JSON blob per row, see
|
||||
* src/custom-avatar-items-db.ts) minus `PurchaseInfo`, which is a store-side projection and
|
||||
* never stored. A record whose id is already present is REPLACED, so re-running an import
|
||||
* with corrected data lands; a `DELETE` is not issued, so player-made items and previously
|
||||
* imported items the new export lacks are left alone.
|
||||
* For a load that needs no migration and no deploy — the usual case — use
|
||||
* `just admin cai-load` instead, which runs the same statements straight against a database.
|
||||
* The statements come from src/custom-avatar-items-load.ts either way: each record stored as
|
||||
* the row's JSON minus `PurchaseInfo` and with `CreatorAccountId` forced to the Coach account
|
||||
* (1), an existing id REPLACED, nothing deleted.
|
||||
*
|
||||
* The saves name assetbundles and thumbnails by bare filename; the migration does nothing
|
||||
* about those files, which have to be served from wherever the client fetches them.
|
||||
@@ -21,11 +21,10 @@
|
||||
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
|
||||
interface Record_ {
|
||||
CustomAvatarItemId: string
|
||||
PurchaseInfo?: unknown
|
||||
[key: string]: unknown
|
||||
}
|
||||
import {
|
||||
customAvatarItemUpsertStatements,
|
||||
readCustomAvatarItemExport,
|
||||
} from '../src/custom-avatar-items-load'
|
||||
|
||||
const [input, number] = process.argv.slice(2)
|
||||
if (!input || !number || !/^\d{4}$/.test(number)) {
|
||||
@@ -33,34 +32,18 @@ if (!input || !number || !/^\d{4}$/.test(number)) {
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
const parsed: unknown = JSON.parse(readFileSync(input, 'utf8'))
|
||||
const records: Record_[] = Array.isArray(parsed)
|
||||
? parsed
|
||||
: parsed && typeof parsed === 'object' && Array.isArray((parsed as { Results?: unknown }).Results)
|
||||
? (parsed as { Results: Record_[] }).Results
|
||||
: [parsed as Record_]
|
||||
|
||||
const seen = new Set<string>()
|
||||
const lines: string[] = [
|
||||
// Tolerate the BOM a .NET export carries (`JSON.parse` rejects U+FEFF).
|
||||
const records = readCustomAvatarItemExport(
|
||||
JSON.parse(readFileSync(input, 'utf8').replace(/^/, ''))
|
||||
)
|
||||
const lines = [
|
||||
`-- Seed first-party custom avatar items (${records.length} records) from ${input.split('/').pop()}.`,
|
||||
'-- Generated by apps/api/scripts/import-custom-avatar-items.ts — do not edit by hand.',
|
||||
'-- Each record is the official `CustomAvatarItem` verbatim; an existing id is replaced.',
|
||||
'-- Each record is the official `CustomAvatarItem`, filed under the Coach account (CreatorAccountId 1);',
|
||||
'-- an existing id is replaced.',
|
||||
'',
|
||||
...customAvatarItemUpsertStatements(records),
|
||||
]
|
||||
for (const record of records) {
|
||||
if (typeof record.CustomAvatarItemId !== 'string' || record.CustomAvatarItemId === '') {
|
||||
throw new Error(`record without a CustomAvatarItemId: ${JSON.stringify(record).slice(0, 80)}`)
|
||||
}
|
||||
if (seen.has(record.CustomAvatarItemId)) {
|
||||
throw new Error(`duplicate CustomAvatarItemId in export: ${record.CustomAvatarItemId}`)
|
||||
}
|
||||
seen.add(record.CustomAvatarItemId)
|
||||
const { PurchaseInfo: _purchaseInfo, ...stored } = record
|
||||
const json = JSON.stringify(stored).replaceAll("'", "''")
|
||||
lines.push(
|
||||
`INSERT INTO custom_avatar_item (data) VALUES ('${json}') ON CONFLICT(custom_avatar_item_id) DO UPDATE SET data = excluded.data;`
|
||||
)
|
||||
}
|
||||
|
||||
const out = decodeURIComponent(
|
||||
new URL(`../migrations/${number}_import_custom_avatar_items.sql`, import.meta.url).pathname
|
||||
|
||||
@@ -27,6 +27,8 @@
|
||||
* as JSON by 0022_custom_avatar_item_json.sql, applied under its own `migrations_table`).
|
||||
*/
|
||||
|
||||
import { COACH_ACCOUNT_ID } from './custom-avatar-items-load'
|
||||
|
||||
/** Schema DDL (mirror of migrations/0022_custom_avatar_item_json.sql). */
|
||||
export const SCHEMA_DDL: string[] = [
|
||||
`CREATE TABLE IF NOT EXISTS custom_avatar_item (
|
||||
@@ -336,9 +338,10 @@ export async function listHotCustomAvatarItems(
|
||||
|
||||
/**
|
||||
* The "Coach" system account — this server's stock content is authored by it, the same id the
|
||||
* `econ` worker attributes a self-buy or an anonymous gift to.
|
||||
* `econ` worker attributes a self-buy or an anonymous gift to. Lives in the loader module so
|
||||
* the CLI can name it without Workers types; re-exported here for the worker-side callers.
|
||||
*/
|
||||
export const COACH_ACCOUNT_ID = 1
|
||||
export { COACH_ACCOUNT_ID }
|
||||
|
||||
/** What `GET /api/customAvatarItems/v1/search` narrows the catalog by. */
|
||||
export interface CustomAvatarItemSearch {
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* Turning an export of custom avatar items into `custom_avatar_item` rows — the loader half of
|
||||
* the table, used by `runx admin cai-load` (in @repo/tools) and by the migration generator
|
||||
* (scripts/import-custom-avatar-items.ts) rather than by the worker.
|
||||
*
|
||||
* Separate from `custom-avatar-items-db.ts` for the reason `catalog-load.ts` is separate from
|
||||
* `catalog-db.ts` in econ: that module types its queries with `D1Database`, a Workers type,
|
||||
* and the loader runs in a plain Node CLI that has no such types. Everything here is pure data
|
||||
* mapping with no imports, so both sides can use it.
|
||||
*
|
||||
* A row is the record as JSON (see 0022_custom_avatar_item_json.sql), so a load is the export
|
||||
* verbatim: no per-field mapping, and a field the source adds later rides along. Two things
|
||||
* change on the way in: `PurchaseInfo`, a store-side projection the worker fills at read time,
|
||||
* is stripped, and `CreatorAccountId` is FORCED to the Coach account — see
|
||||
* {@link customAvatarItemRowLiteral}.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The "Coach" system account — this server's stock content is authored by it, the same id the
|
||||
* `econ` worker attributes a self-buy or an anonymous gift to. Defined here rather than in
|
||||
* `custom-avatar-items-db.ts` (which re-exports it) so the loader can name it without pulling
|
||||
* in Workers types.
|
||||
*/
|
||||
export const COACH_ACCOUNT_ID = 1
|
||||
|
||||
/** The least an export record must carry to be a row. Everything else is kept as-is. */
|
||||
export interface CustomAvatarItemExportRecord {
|
||||
CustomAvatarItemId: string
|
||||
PurchaseInfo?: unknown
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
/**
|
||||
* The records in an export, whether it is a bare JSON array, a `{ Results: [...] }` page, or a
|
||||
* single record. Refuses a record without an id and a repeated id — a repeat in one upsert
|
||||
* statement is an error at the database, and a repeat across statements would silently keep
|
||||
* whichever came last, which is a defect in the export rather than a choice to make for it.
|
||||
*/
|
||||
export function readCustomAvatarItemExport(parsed: unknown): CustomAvatarItemExportRecord[] {
|
||||
const records: unknown[] = Array.isArray(parsed)
|
||||
? parsed
|
||||
: parsed &&
|
||||
typeof parsed === 'object' &&
|
||||
Array.isArray((parsed as { Results?: unknown }).Results)
|
||||
? (parsed as { Results: unknown[] }).Results
|
||||
: [parsed]
|
||||
const seen = new Set<string>()
|
||||
return records.map((record, i) => {
|
||||
if (!record || typeof record !== 'object' || Array.isArray(record)) {
|
||||
throw new Error(`record ${i} is not an object`)
|
||||
}
|
||||
const id = (record as { CustomAvatarItemId?: unknown }).CustomAvatarItemId
|
||||
if (typeof id !== 'string' || id === '') {
|
||||
throw new Error(`record ${i} has no CustomAvatarItemId`)
|
||||
}
|
||||
if (seen.has(id)) throw new Error(`duplicate CustomAvatarItemId in export: ${id}`)
|
||||
seen.add(id)
|
||||
return record as CustomAvatarItemExportRecord
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* One record as the SQL string literal its row holds: the JSON, minus `PurchaseInfo`, with
|
||||
* `CreatorAccountId` overridden to {@link COACH_ACCOUNT_ID} whatever the export said. An
|
||||
* imported item is first-party content by definition, and the creator id is what makes it so
|
||||
* here: the storefront tab finds stock items by `creator_account_id = 1`
|
||||
* (`includeCoachItems=True`), and a sale pays the creator. An export from another server
|
||||
* carries THAT server's creator ids — an item filed under one of those would vanish from the
|
||||
* storefront into the user-generated tab, and its price would be paid to whichever local
|
||||
* account happened to share the number.
|
||||
*/
|
||||
export function customAvatarItemRowLiteral(record: CustomAvatarItemExportRecord): string {
|
||||
const { PurchaseInfo: _purchaseInfo, ...stored } = record
|
||||
const row = { ...stored, CreatorAccountId: COACH_ACCOUNT_ID }
|
||||
return `'${JSON.stringify(row).replaceAll("'", "''")}'`
|
||||
}
|
||||
|
||||
/**
|
||||
* The largest statement a load builds, in bytes of SQL. D1 caps a single statement at 100 KB,
|
||||
* and a record with three built saves runs to a few KB, so rows are grouped by size rather
|
||||
* than by count — a fixed count would fit the small records and overflow on the large ones.
|
||||
*/
|
||||
export const CUSTOM_AVATAR_ITEM_LOAD_STATEMENT_BYTES = 64 * 1024
|
||||
|
||||
/**
|
||||
* The statements that upsert `records` into `custom_avatar_item`, each a multi-row INSERT
|
||||
* within {@link CUSTOM_AVATAR_ITEM_LOAD_STATEMENT_BYTES}. An id already in the table has its
|
||||
* row REPLACED (`ON CONFLICT ... DO UPDATE`), so a re-run with a corrected export lands and a
|
||||
* re-run with the same one changes nothing. Nothing is deleted: rows the export does not
|
||||
* mention — the players' own shirts among them — are left alone.
|
||||
*/
|
||||
export function customAvatarItemUpsertStatements(
|
||||
records: CustomAvatarItemExportRecord[],
|
||||
maxBytes = CUSTOM_AVATAR_ITEM_LOAD_STATEMENT_BYTES
|
||||
): string[] {
|
||||
const head = 'INSERT INTO custom_avatar_item (data) VALUES\n'
|
||||
const tail = '\nON CONFLICT(custom_avatar_item_id) DO UPDATE SET data = excluded.data;'
|
||||
const statements: string[] = []
|
||||
let rows: string[] = []
|
||||
let bytes = 0
|
||||
const flush = (): void => {
|
||||
if (rows.length === 0) return
|
||||
statements.push(head + rows.map((r) => `\t(${r})`).join(',\n') + tail)
|
||||
rows = []
|
||||
bytes = 0
|
||||
}
|
||||
for (const record of records) {
|
||||
const literal = customAvatarItemRowLiteral(record)
|
||||
// A single oversized record still gets its own statement rather than being dropped.
|
||||
if (rows.length > 0 && bytes + literal.length > maxBytes) flush()
|
||||
rows.push(literal)
|
||||
bytes += literal.length + 4
|
||||
}
|
||||
flush()
|
||||
return statements
|
||||
}
|
||||
Reference in New Issue
Block a user