[econ] fix some storefronts

CLI.md

@@ -88,6 +88,25 @@ This command targets the `balance` table, so it runs under `apps/econ`'s wrangle
 with `--local` that is econ's dev D1 state, which needs both `balance` and a migrated
 `account` table in it.
 
+### `catalog load` — reload the item catalog
+
+```sh
+just catalog-load                 # sf3-2025.json → local (merge)
+just catalog-load --remote        # production
+just catalog-load --replace       # also drop rows the file no longer lists
+just catalog-load --dry-run       # build and validate, write nothing
+bun runx catalog check            # validate the store file without touching a database
+```
+
+Fills econ's `catalog` table from `apps/econ/static/storefronts/sf3-2025.json`, the 2025
+store as the game is served it, and nothing else. Re-run it whenever that file changes (after
+`runx storefront build`). Each row's `catalog_id` is its listing's `PurchasableItemId` and its
+`listing` column the listing verbatim, which is what
+`POST /api/storefronts/v1/purchasableitems` serves. Avatar items, consumables, skins and
+first-party custom avatar items all load; loot boxes, token bundles and consumable packs do
+not. A merge leaves rows the file no longer lists in place with no `catalog_id` or `listing`;
+`--replace` deletes them. The inventory does not read this table, so either is safe.
+
 ### `cai-load` — load the first-party custom avatar items
 
 ```sh

Justfile

@@ -134,3 +134,12 @@ reload-plus *args:
 [no-cd]
 cai-load *args:
   bun runx admin cai-load "$@"
+
+# Merges by default; --replace drops rows the file no longer lists. Defaults to --local; pass
+# --remote for production, --dry-run to only validate. Run it whenever sf3-2025.json changes.
+# Reload the item catalog (econ `catalog` table) from apps/econ/static/storefronts/sf3-2025.json
+[group('4. utility')]
+[positional-arguments]
+[no-cd]
+catalog-load *args:
+  bun runx catalog load "$@"

apps/econ/migrations/0028_catalog_listing.sql

@@ -0,0 +1,15 @@
+-- The served store listing behind a catalog row, verbatim as JSON: the `StoreItems[]` entry
+-- of `static/storefronts/sf3-2025.json` whose `PurchasableItemId` is the row's `catalog_id`.
+--
+-- `POST /api/storefronts/v1/purchasableitems` answers with whole listings — prices, item set,
+-- `AvatarItemInfo` and the rest — which the catalog's own columns do not carry. Keeping the
+-- listing on the row lets that lookup be one indexed query instead of fetching and parsing the
+-- 4 MB storefront file per request.
+--
+-- Filled by `runx catalog load`, like every other column. NULL on a row the served store does
+-- not list (the same rows whose `catalog_id` is NULL), and until the first load after this
+-- migration.
+--
+-- Its own migration because 0015 and 0016 are already applied.
+
+ALTER TABLE catalog ADD COLUMN listing TEXT;

apps/econ/src/catalog-db.ts

@@ -1,6 +1,6 @@
 import { CatalogKind } from './catalog-load'
 
-import type { CatalogKindValue } from './catalog-load'
+import type { CatalogKindValue, StoreListing } from './catalog-load'
 
 /**
  * The item CATALOG on the shared `recflare` D1 database — every avatar item, consumable and
@@ -96,7 +96,8 @@ export const CATALOG_SCHEMA_DDL: string[] = [
 		tag_list TEXT,
 		created_at TEXT,
 		prefab_name TEXT,
-		unlocked_level INTEGER
+		unlocked_level INTEGER,
+		listing TEXT
 	)`,
 	// The search index. Folded, because a name search is case-insensitive and SQLite's LIKE is
 	// only case-insensitive for ASCII — which these names are not all of.
@@ -138,6 +139,12 @@ export interface CatalogRow {
 	created_at: string | null
 	prefab_name: string | null
 	unlocked_level: number | null
+	/**
+	 * The served store listing behind the row — its `sf3-2025.json` `StoreItems[]` entry,
+	 * verbatim, as JSON (migration 0028). NULL where `catalog_id` is: a row the store does not
+	 * list has no listing. Read it through {@link getStoreListings}.
+	 */
+	listing: string | null
 }
 
 /**
@@ -271,6 +278,35 @@ export async function getCatalogItemById(
 		.first<CatalogRow>()
 }
 
+/**
+ * Store listings by `PurchasableItemId` (the `catalog_id`), as the served store lists them —
+ * what `POST /api/storefronts/v1/purchasableitems` answers. In the order asked; an unknown id,
+ * a repeat, or a row without a listing is skipped rather than left as a hole, so the result may
+ * be SHORTER than the input and must not be read positionally.
+ *
+ * Every kind resolves — avatar items, consumables, skins and custom avatar items alike — and the
+ * listing is served exactly as the store lists it. Only listings that load as no row at all
+ * (loot boxes, token bundles, consumable packs; see `ListingKind`) are unknown here.
+ */
+export async function getStoreListings(db: D1Database, ids: number[]): Promise<StoreListing[]> {
+	const unique = [...new Set(ids)]
+	if (unique.length === 0) return []
+	// One bound JSON array rather than a placeholder per id: D1 caps a statement at 100 bound
+	// variables, and nothing stops the client asking for more.
+	const { results } = await db
+		.prepare(
+			`SELECT catalog_id, listing FROM catalog
+			 WHERE catalog_id IN (SELECT value FROM json_each(?1)) AND listing IS NOT NULL`
+		)
+		.bind(JSON.stringify(unique))
+		.all<{ catalog_id: number; listing: string }>()
+	const byId = new Map(results.map((r) => [r.catalog_id, r.listing]))
+	return unique.flatMap((id) => {
+		const listing = byId.get(id)
+		return listing === undefined ? [] : [JSON.parse(listing) as StoreListing]
+	})
+}
+
 /**
  * Resolve many keys at once, in the order asked; unknown keys are skipped rather than left as
  * holes, so the result may be SHORTER than the input and must not be read positionally. One
@@ -350,8 +386,8 @@ export async function getAvatarItemsByTag(
 	tag: string
 ): Promise<CatalogAvatarItem[]> {
 	const { results } = await db
-		.prepare('SELECT * FROM catalog WHERE tag_list = ?1 ORDER BY friendly_name')
+		.prepare('SELECT * FROM catalog WHERE kind = ?1 AND tag_list = ?2 ORDER BY friendly_name')
-		.bind(tag)
+		.bind(CatalogKind.AvatarItem, tag)
 		.all<CatalogRow>()
 	return results.map(toCatalogAvatarItem)
 }

apps/econ/src/catalog-load.ts

@@ -29,6 +29,13 @@ export const CatalogKind = {
 	Skin: 'skin',
 	/** A consumable: food, a potion, a KO icon. Its `item_key` is the `ConsumableItemDesc`. */
 	Consumable: 'consumable',
+	/**
+	 * A first-party custom avatar item the store lists. Its `item_key` is the
+	 * `CustomAvatarItemId`. The ITEM lives in `api`'s `custom_avatar_item` table (`just cai-load`)
+	 * and is resolved there; the row here exists for its LISTING, so a `PurchasableItemId` lookup
+	 * answers for every item the store sells. Only the shared columns are filled.
+	 */
+	CustomAvatarItem: 'custom_avatar_item',
 } as const
 
 export type CatalogKindValue = (typeof CatalogKind)[keyof typeof CatalogKind]
@@ -114,15 +121,13 @@ export interface SkinCapture {
 /**
  * What a store listing SELLS — the catalog kind it loads as, or why it loads as nothing.
  *
- * `custom_avatar_item` listings are first-party UGC: they live in the `custom_avatar_item`
+ * `query` is a loot box and `currency` a token bundle —
- * table (`just cai-load`), not here. `query` is a loot box and `currency` a token bundle —
  * neither is an item. `unknown` carries no id at all (two expo items the dump prices at
  * nothing). `consumable_pack` is assigned by the load rather than by {@link listingKind}: a
  * consumable the store sells singly AND as a pack ("Disco Dance Break", "… 3-Pack") is one
  * consumable listed twice, and the catalog is of things, not of listings.
  */
-export type ListingKind =
+export type ListingKind = CatalogKindValue | 'query' | 'currency' | 'unknown' | 'consumable_pack'
-	CatalogKindValue | 'custom_avatar_item' | 'query' | 'currency' | 'unknown' | 'consumable_pack'
 
 /** Classify one listing by which of its item ids is set. */
 export function listingKind(listing: StoreListing): ListingKind {
@@ -131,7 +136,7 @@ export function listingKind(listing: StoreListing): ListingKind {
 	if (drop.ConsumableItemDesc !== '') return CatalogKind.Consumable
 	if (drop.EquipmentModificationGuid !== '') return CatalogKind.Skin
 	if (drop.CustomAvatarItemId !== null && drop.CustomAvatarItemId !== '') {
-		return 'custom_avatar_item'
+		return CatalogKind.CustomAvatarItem
 	}
 	if (drop.IsQuery) return 'query'
 	if (drop.Currency > 0) return 'currency'
@@ -270,6 +275,7 @@ export const CATALOG_INSERT_COLUMNS = [
 	'created_at',
 	'prefab_name',
 	'unlocked_level',
+	'listing',
 ] as const
 
 /** A value bound into a load's INSERT. `undefined` is a key the capture omitted. */
@@ -332,6 +338,7 @@ function avatarItemRow(listing: StoreListing): CatalogLoadRow {
 			null,
 			null,
 			null,
+			JSON.stringify(listing),
 		],
 	}
 }
@@ -359,6 +366,7 @@ function consumableRow(listing: StoreListing): CatalogLoadRow {
 			null,
 			null,
 			null,
+			JSON.stringify(listing),
 		],
 	}
 }
@@ -386,6 +394,39 @@ function listedSkinRow(listing: StoreListing): CatalogLoadRow {
 			null,
 			drop.EquipmentPrefabName,
 			null,
+			JSON.stringify(listing),
+		],
+	}
+}
+
+/**
+ * A listed first-party custom avatar item: the shared columns, keyed by its
+ * `CustomAvatarItemId`. The record itself is `api`'s; this row carries the listing.
+ */
+function customAvatarItemRow(listing: StoreListing): CatalogLoadRow {
+	const drop = listing.GiftDrop
+	const guid = drop.CustomAvatarItemId as string
+	return {
+		key: guid,
+		id: listing.PurchasableItemId,
+		label: `${drop.FriendlyName} (custom avatar item)`,
+		values: [
+			guid,
+			listing.PurchasableItemId,
+			CatalogKind.CustomAvatarItem,
+			drop.FriendlyName,
+			drop.Tooltip,
+			drop.Rarity,
+			ALL_PLATFORMS,
+			drop.ThumbnailImageName,
+			null,
+			null,
+			null,
+			drop.TagList,
+			null,
+			null,
+			null,
+			JSON.stringify(listing),
 		],
 	}
 }
@@ -445,6 +486,9 @@ export function buildCatalogLoad(listings: StoreListing[]): CatalogLoad {
 			case CatalogKind.Skin:
 				push(listedSkinRow(listing))
 				break
+			case CatalogKind.CustomAvatarItem:
+				push(customAvatarItemRow(listing))
+				break
 			default:
 				skipped[kind] = (skipped[kind] ?? 0) + 1
 		}

apps/econ/src/econ.app.ts

@@ -59,7 +59,7 @@ import {
 	isSpendable,
 	spendCurrency,
 } from './balance-db'
-import { getCatalogItem, toCatalogAvatarItem } from './catalog-db'
+import { getCatalogItem, getStoreListings, toCatalogAvatarItem } from './catalog-db'
 // `LEGACY_CLIENT_BUILD` is shared with the storefront generator rather than restated: it picks
 // which store FILE a caller is served here, and is what the generator's help names.
 import { CatalogKind, LEGACY_CLIENT_BUILD } from './catalog-load'
@@ -130,6 +130,7 @@ import {
 	OPTIONAL_AUTHED,
 	OwnsRoomKeysRequest,
 	OwnsRoomKeysResponse,
+	PurchasableItemsRequest,
 	PurchaseRoomCurrencyEnvelope,
 	PurchaseRoomCurrencyRequest,
 	PurchaseRoomKeyWithCurrencyRequest,
@@ -5061,6 +5062,45 @@ const app = new Hono<App>({ strict: false })
 		}
 	)
 
+	// Store listings by `PurchasableItemId` — `{ PurchasableItemIds: [2594], GiftDropIds: null }`.
+	// Answers a bare array of the listings exactly as the served 2025 store (`sf3-2025.json`)
+	// lists them, read from the `catalog` table's `listing` column rather than the 4 MB file
+	// (a row's `catalog_id` IS its `PurchasableItemId`). Every item kind resolves; unknown ids
+	// are dropped, so the answer is in request order but not positional. `GiftDropIds` has only
+	// been seen null and is ignored. No auth: a listing is what the public store shows anyone.
+	.post(
+		'/api/storefronts/v1/purchasableitems',
+		describeRoute({
+			tags: ['Storefront'],
+			summary: 'Store listings by PurchasableItemId',
+			description: [
+				'Resolves `PurchasableItemIds[]` against the `catalog` table and answers each listing',
+				'verbatim as the served 2025 store (`sf3-2025.json`) lists it — `GiftDrop`, `Prices`,',
+				'`SubscriberPrices` and the rest. Avatar items, consumables, skins and custom avatar',
+				'items all resolve. Unknown ids (and repeats) are dropped, so the response follows',
+				'request order but is never a positional match. `GiftDropIds` is ignored.',
+			].join(' '),
+			requestBody: jsonBody(PurchasableItemsRequest, 'The listings to resolve'),
+			responses: {
+				200: json(JsonArray, 'The resolved listings (unknown ids omitted)'),
+				400: json(ErrorResponse, 'Malformed body'),
+			},
+		}),
+		async (c) => {
+			const body = (await c.req.json().catch(() => null)) as Record<string, unknown> | null
+			if (!body || typeof body !== 'object')
+				return c.json({ error: 'A JSON body is required' }, 400)
+			const raw = body.PurchasableItemIds
+			if (raw !== undefined && raw !== null && !Array.isArray(raw)) {
+				return c.json({ error: 'PurchasableItemIds must be an array' }, 400)
+			}
+			const ids = ((raw as unknown[] | null | undefined) ?? []).filter((id): id is number =>
+				Number.isInteger(id)
+			)
+			return c.json(await getStoreListings(c.env.DB, ids))
+		}
+	)
+
 	// The consumable DEFINITIONS the client knows how to use (`?supportedVersion=2`), as
 	// opposed to the ones a player owns. Served verbatim from `static/consumables.json`, the
 	// live game's list; `supportedVersion` is accepted and unused. A row's `ConfigurationData`

apps/econ/src/openapi.ts

@@ -633,6 +633,18 @@ export const ItemPurchaseInfosRequest = z.object({
 	),
 })
 
+/** `POST /api/storefronts/v1/purchasableitems` — store listings by id. */
+export const PurchasableItemsRequest = z.object({
+	PurchasableItemIds: z
+		.array(z.number().int())
+		.nullable()
+		.describe('The listings to resolve, by `PurchasableItemId`'),
+	GiftDropIds: z
+		.array(z.number().int())
+		.nullable()
+		.describe('Sent by the client (observed null); accepted and ignored'),
+})
+
 /** One `Prices[]` entry: what the item costs in one currency, and any sale on top. */
 export const ItemPriceDto = z.object({
 	CurrencyType: z.number().int().describe('2 = RecCenterTokens — what UGC items are priced in'),

apps/econ/src/test/integration/api.test.ts

@@ -42,6 +42,7 @@ import { NotificationType } from '../../../../notify/src/notification-types'
 // SQL, not modules: they are never executed here, only read.
 import catalogStructureSql from '../../../migrations/0015_catalog.sql?raw'
 import catalogIdSql from '../../../migrations/0016_catalog_id.sql?raw'
+import catalogListingSql from '../../../migrations/0028_catalog_listing.sql?raw'
 import skinsJson from '../../../static/db/skins.json'
 // The 2025 store DUMP, the source `runx storefront build` and `runx catalog load` both read.
 import storeDump from '../../../static/db/Watch_EnumValue_3.json'
@@ -3707,6 +3708,57 @@ describe('econ endpoints', () => {
 		])
 	})
 
+	test('POST /api/storefronts/v1/purchasableitems answers the served listings verbatim', async () => {
+		// One listing of each kind the catalog holds, loaded exactly as `runx catalog load` would.
+		const pick = (pred: (l: StoreListing) => boolean) => storeListings.find(pred)!
+		const wanted = [
+			pick((l) => l.PurchasableItemId === 2594), // an avatar item
+			pick((l) => l.GiftDrop.ConsumableItemDesc !== ''),
+			pick((l) => l.GiftDrop.EquipmentModificationGuid !== ''),
+			pick((l) => (l.GiftDrop.CustomAvatarItemId ?? '') !== ''),
+		]
+		const ids = new Set(wanted.map((l) => l.PurchasableItemId))
+		const rows = buildCatalogLoad(storeListings).rows.filter((r) => ids.has(r.id))
+		expect(rows).toHaveLength(wanted.length)
+		await env.DB.batch(
+			rows.map((row) =>
+				env.DB.prepare(
+					`INSERT OR REPLACE INTO catalog (${CATALOG_INSERT_COLUMNS.join(', ')})
+					 VALUES (${CATALOG_INSERT_COLUMNS.map((_, i) => `?${i + 1}`).join(', ')})`
+				).bind(...row.values.map((v) => v ?? null))
+			)
+		)
+
+		// Request order, unknown ids and repeats dropped, `GiftDropIds` ignored. No token needed.
+		const order = [...wanted].reverse().map((l) => l.PurchasableItemId)
+		const res = await exports.default.fetch(`${ORIGIN}/api/storefronts/v1/purchasableitems`, {
+			method: 'POST',
+			headers: { 'content-type': 'application/json' },
+			body: JSON.stringify({
+				PurchasableItemIds: [order[0], 12_345_678, ...order, order[1]],
+				GiftDropIds: null,
+			}),
+		})
+		expect(res.status).toBe(200)
+		const body = (await res.json()) as StoreListing[]
+		expect(body).toEqual(order.map((id) => storeListings.find((l) => l.PurchasableItemId === id)))
+		expect(body.at(-1)?.GiftDrop.FriendlyName).toBe('Hot Dog Suit (Ketchup)')
+
+		// Null ids answer an empty list; a non-array is a malformed body.
+		const none = await exports.default.fetch(`${ORIGIN}/api/storefronts/v1/purchasableitems`, {
+			method: 'POST',
+			headers: { 'content-type': 'application/json' },
+			body: JSON.stringify({ PurchasableItemIds: null, GiftDropIds: null }),
+		})
+		expect(await none.json()).toEqual([])
+		const bad = await exports.default.fetch(`${ORIGIN}/api/storefronts/v1/purchasableitems`, {
+			method: 'POST',
+			headers: { 'content-type': 'application/json' },
+			body: JSON.stringify({ PurchasableItemIds: 2594 }),
+		})
+		expect(bad.status).toBe(400)
+	})
+
 	test('POST /api/items/purchaseInfos 400s without Ids and 401s without a token', async () => {
 		const bad = await exports.default.fetch(`${ORIGIN}/api/items/purchaseInfos`, {
 			method: 'POST',
@@ -7571,6 +7623,7 @@ describe('econ endpoints', () => {
 			'POST /api/roomkeys/v1/create',
 			'POST /api/roomkeys/v1/owns/bulk',
 			'POST /api/storefronts/v1/PurchaseRoomKeyWithCurrency',
+			'POST /api/storefronts/v1/purchasableitems',
 			'POST /api/storefronts/v2/buyItem',
 			'POST /api/storefronts/v3/buyInvention',
 			'POST /api/timedCurrencyGrants/update',
@@ -7893,7 +7946,7 @@ describe('catalog', () => {
 		// separate because 0015 was already applied, and an edit there would never re-run — which
 		// is exactly the drift this test exists to catch. `CATALOG_SCHEMA_DDL` declares the end
 		// state in one CREATE, so it is compared against the pair.
-		const migrations = `${catalogStructureSql}\n${catalogIdSql}`
+		const migrations = `${catalogStructureSql}\n${catalogIdSql}\n${catalogListingSql}`
 
 		// Compared on identifiers rather than text, since the two are formatted differently, and
 		// `catalog_id` arrives via ALTER rather than inside the CREATE.
@@ -7904,6 +7957,7 @@ describe('catalog', () => {
 		expect(catalogStructureSql).toContain('item_key TEXT PRIMARY KEY')
 		expect(CATALOG_SCHEMA_DDL[0]).toContain('item_key TEXT PRIMARY KEY')
 		expect(catalogIdSql).toContain('ALTER TABLE catalog ADD COLUMN catalog_id INTEGER')
+		expect(catalogListingSql).toContain('ALTER TABLE catalog ADD COLUMN listing TEXT')
 		for (const index of [
 			'idx_catalog_name',
 			'idx_catalog_prefab',
@@ -7968,13 +8022,27 @@ describe('catalog', () => {
 		)
 		expect(row.values[CATALOG_INSERT_COLUMNS.indexOf('rarity')]).toBe(both.GiftDrop.Rarity)
 
-		// What the store lists that is NOT a catalog item is counted, not loaded: first-party
+		// First-party custom items load as rows of their own, keyed by guid: the item is
-		// custom items (the `custom_avatar_item` table's), loot boxes, token bundles — and the
+		// `api`'s `custom_avatar_item` record, but its LISTING has to resolve by number too.
-		// PACKS, a consumable listed again under another number with a count. Every listing is
+		expect(kinds.filter((k) => k === 'custom_avatar_item')).toHaveLength(
-		// accounted for: a row, a skipped count, or nothing else.
+			listed((l) => l.GiftDrop.CustomAvatarItemId ?? '')
-		expect(skipped.custom_avatar_item).toBe(
-			storeListings.filter((l) => l.GiftDrop.CustomAvatarItemId).length
 		)
+
+		// Every row carries its listing verbatim, so the store's own JSON comes back out.
+		const listingAt = CATALOG_INSERT_COLUMNS.indexOf('listing')
+		const byPurchasableId = new Map(storeListings.map((l) => [l.PurchasableItemId, l]))
+		expect(
+			rows.every(
+				(r) =>
+					r.values[listingAt] === JSON.stringify(byPurchasableId.get(r.id)) &&
+					r.values[listingAt] !== undefined
+			)
+		).toBe(true)
+
+		// What the store lists that is NOT a catalog item is counted, not loaded: loot boxes,
+		// token bundles — and the PACKS, a consumable listed again under another number with a
+		// count. Every listing is accounted for: a row, a skipped count, or nothing else.
+		expect(skipped.custom_avatar_item).toBeUndefined()
 		expect(skipped.query).toBe(storeListings.filter((l) => l.GiftDrop.IsQuery).length)
 		expect(skipped.consumable_pack).toBeGreaterThan(0)
 		expect(
@@ -8031,7 +8099,12 @@ describe('catalog', () => {
 			const drop = byId.get(row.id)?.GiftDrop
 			expect(drop, String(row.id)).toBeDefined()
 			expect(
-				[drop!.AvatarItemDesc, drop!.ConsumableItemDesc, drop!.EquipmentModificationGuid],
+				[
+					drop!.AvatarItemDesc,
+					drop!.ConsumableItemDesc,
+					drop!.EquipmentModificationGuid,
+					drop!.CustomAvatarItemId,
+				],
 				String(row.id)
 			).toContain(row.key)
 		}

apps/www/src/staff.ts

@@ -1302,8 +1302,11 @@ async function findGiftableItem(db: D1Database, requested: string): Promise<Gift
 		getCatalogItem(db, requested),
 	])
 	const item = custom !== null && custom.Accessibility !== 0 ? custom : null
-	const row =
+	const found =
 		byKey ?? (/^\d{1,9}$/.test(requested) ? await getCatalogItemById(db, Number(requested)) : null)
+	// A custom avatar item's catalog row only carries its store listing; the item itself is the
+	// `custom_avatar_item` record found above, so the row must not make its guid `ambiguous`.
+	const row = found?.kind === CatalogKind.CustomAvatarItem ? null : found
 
 	if (item !== null && row !== null) return { kind: 'ambiguous' }
 	if (item !== null) return { kind: 'custom_item', item }

packages/tools/src/cmd/catalog.cmd.ts

@@ -53,10 +53,12 @@ export const STORE_DUMP = 'apps/econ/static/db/Watch_EnumValue_3.json'
 export const SKINS = 'apps/econ/static/db/skins.json'
 
 /**
- * Rows per INSERT. Batched because thousands of single-row statements parse far more slowly
+ * The largest INSERT a load builds, in bytes of SQL. Batched because thousands of single-row
- * than a few dozen multi-row ones, and D1 charges per statement on the remote path.
+ * statements parse far more slowly than a few dozen multi-row ones, and D1 charges per statement
+ * on the remote path; capped by SIZE rather than row count because D1 refuses a statement over
+ * 100 KB, and each row carries its whole store listing (~1.2 KB, up to ~1.4 KB).
  */
-const CHUNK = 100
+const STATEMENT_BYTES = 64 * 1024
 
 /**
  * One SQL literal. Booleans become 1/0 (SQLite has no boolean), `undefined` is treated as the
@@ -271,16 +273,27 @@ const load = new Command('load')
 			// afterwards is visibly a row the store no longer lists.
 			statements.push('UPDATE catalog SET catalog_id = NULL;')
 		}
-		for (let start = 0; start < rows.length; start += CHUNK) {
+		let batch: string[] = []
-			const batch = rows.slice(start, start + CHUNK)
+		let batchBytes = 0
+		const flush = (): void => {
+			if (batch.length === 0) return
 			statements.push(
 				`INSERT INTO catalog (${CATALOG_INSERT_COLUMNS.join(', ')}) VALUES\n` +
-					batch.map((r) => `\t(${r.values.map(lit).join(', ')})`).join(',\n') +
+					batch.join(',\n') +
 					// Harmless after a DELETE (nothing can conflict), and kept there anyway so both
 					// paths run the identical statement rather than two shapes that could diverge.
 					`\nON CONFLICT(item_key) DO UPDATE SET ${CONFLICT_UPDATE};`
 			)
+			batch = []
+			batchBytes = 0
 		}
+		for (const row of rows) {
+			const tuple = `\t(${row.values.map(lit).join(', ')})`
+			if (batch.length > 0 && batchBytes + tuple.length > STATEMENT_BYTES) flush()
+			batch.push(tuple)
+			batchBytes += tuple.length + 2
+		}
+		flush()
 		// No numbering step after the inserts. A row's `catalog_id` is its listing's
 		// `PurchasableItemId` in sf3-2025.json and nothing else; a row a merge keeps that the file
 		// no longer lists has no such number, so it stays NULL (every reader filters on

packages/tools/src/d1.ts

@@ -95,6 +95,22 @@ async function runD1(worker: string, extraArgs: string[], remote: boolean): Prom
 		// dependency of @repo/tools, so it's not on this process's PATH).
 		const out = await $`pnpm exec wrangler ${args}`.quiet()
 		return out.stdout
+	} catch (e) {
+		// `.quiet()` hides wrangler's output, and runx exits on a failed process WITHOUT
+		// printing it — so a D1 error (a constraint, a missing column, an unapplied migration)
+		// surfaced as a bare "exited with code 1". Rethrow with what wrangler actually said.
+		if (e instanceof ProcessOutput) {
+			const said = [e.stderr, e.stdout]
+				.join('\n')
+				.split('\n')
+				.filter((line) => /ERROR|✘|error/i.test(line))
+				.join('\n')
+				.trim()
+			throw new Error(
+				`wrangler d1 execute failed (exit ${e.exitCode}):\n${said || `${e.stderr}\n${e.stdout}`.trim()}`
+			)
+		}
+		throw e
 	} finally {
 		if (cleanup) await cleanup()
 	}