[playersettings] improve migration

packages/tools/src/cmd/admin.cmd.ts

@@ -444,11 +444,50 @@ async function settingsNamespaceIdFromEnv(): Promise<string> {
  * by the `playersettings` worker). A KV value was the player's whole map as a JSON object of
  * strings, which is exactly what the row's `data` holds, so each key becomes one row.
  *
- * Rows that already exist are MERGED, with D1 winning: `json_patch(kv, d1)` keeps every key
- * the live row has and adds the ones only KV knew. That makes the import safe to run after
- * the workers have cut over (a player who has since toggled something keeps it) and to
- * re-run. `--overwrite` makes KV win outright instead. Nothing is deleted on either side.
+ * Rows that already exist are MERGED: KV fills in what D1 lacks, and D1 wins any key it has
+ * moved off the seeded default. A key D1 still holds AT its default (`SEEDED_DEFAULTS`) does
+ * not count as the player's — `GET /playersettings` seeds those for any player with no row, so
+ * a player who signed in after the cut-over but before the import has a fresh row of
+ * defaults, and their KV progress (OOBE, tutorial mask) has to beat it. That makes the import
+ * safe to run after the cut-over and to re-run. `--overwrite` makes KV win outright instead.
+ * Nothing is deleted on either side.
+ *
+ * What it cannot recover is a KV write made after the import read the namespace, to a key the
+ * imported row already holds off-default: D1 keeps the imported value. So run it once before
+ * `just deploy` (the bulk) and once after (to catch anyone who changed something in between).
  */
+/**
+ * What `GET /playersettings` seeds for a player with no row — mirror of `DEFAULT_SETTINGS` in
+ * apps/playersettings/src/default-settings.ts. A D1 value equal to one of these may be the
+ * seed rather than the player's, so the import lets KV replace it.
+ */
+const SEEDED_DEFAULTS: Record<string, string> = {
+	'Recroom.OOBE': '77',
+	TUTORIAL_COMPLETE_MASK: '11',
+	FIRST_TIME_IN_FLAGS: '0',
+}
+
+/**
+ * A KV value as a settings map, or null when it isn't one. Numbers and booleans are kept as
+ * their string form (what a PUT would have stored); a null, object or array value drops just
+ * that key rather than the player's whole map.
+ */
+function kvSettingsMap(value: string | null | undefined): Record<string, string> | null {
+	let parsed: unknown
+	try {
+		parsed = value == null ? null : JSON.parse(value)
+	} catch {
+		return null
+	}
+	if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return null
+	const map: Record<string, string> = {}
+	for (const [k, v] of Object.entries(parsed)) {
+		if (typeof v === 'string') map[k] = v
+		else if (typeof v === 'number' || typeof v === 'boolean') map[k] = String(v)
+	}
+	return map
+}
+
 const settingsImportKv = new Command('settings-import-kv')
 	.description('Copy the old RECFLARE_PLAYER_SETTINGS KV namespace into player_settings (merges)')
 	.option(
@@ -485,7 +524,9 @@ const settingsImportKv = new Command('settings-import-kv')
 			const chunk = keys.slice(i, i + KV_BULK_GET_LIMIT)
 			const file = path.join(os.tmpdir(), `recflare-kv-keys-${Date.now()}-${i}.json`)
 			await fs.writeFile(file, JSON.stringify(chunk))
-			let got: Record<string, { value: string | null }>
+			// `bulk get` answers `{ key: value }`, or `{ key: { value, metadata } }` with
+			// `--with-metadata`; read either.
+			let got: Record<string, string | { value: string | null } | null>
 			try {
 				got = parseWranglerJson(
 					await runKv(['bulk', 'get', file, '--namespace-id', namespaceId], remote)
@@ -495,43 +536,38 @@ const settingsImportKv = new Command('settings-import-kv')
 			}
 			for (const key of chunk) {
 				const accountId = Number(key.slice(SETTINGS_KV_PREFIX.length))
-				const value = got[key]?.value
-				let parsed: unknown = null
-				try {
-					parsed = value == null ? null : JSON.parse(value)
-				} catch {
-					parsed = null
-				}
-				const isMap =
-					parsed !== null &&
-					typeof parsed === 'object' &&
-					!Array.isArray(parsed) &&
-					Object.values(parsed).every((v) => typeof v === 'string')
-				if (!Number.isSafeInteger(accountId) || accountId <= 0 || !isMap) {
+				const raw = got[key]
+				const map = kvSettingsMap(typeof raw === 'string' ? raw : raw?.value)
+				if (!Number.isSafeInteger(accountId) || accountId <= 0 || map === null) {
 					skipped.push(key)
 					continue
 				}
 				// An empty map is what a player with no row reads as; nothing to import.
-				if (Object.keys(parsed as object).length === 0) {
+				if (Object.keys(map).length === 0) {
 					empty++
 					continue
 				}
-				rows.push({ accountId, data: JSON.stringify(parsed) })
+				rows.push({ accountId, data: JSON.stringify(map) })
 			}
 			console.log(`  fetched ${Math.min(i + chunk.length, keys.length)}/${keys.length}`)
 		}
 		for (const key of skipped) console.log(chalk.yellow(`  skipping ${key}: not a settings map`))
 
+		// Merge: start from D1, lay KV over it, then lay back every D1 key that isn't sitting at
+		// its seeded default. (`json_group_object` of no rows is `{}`, which patches nothing.)
+		const d1OwnKeys = `(SELECT json_group_object(s.key, s.value) FROM json_each(player_settings.data) AS s WHERE NOT EXISTS (SELECT 1 FROM json_each('${sqlStr(JSON.stringify(SEEDED_DEFAULTS))}') AS d WHERE d.key = s.key AND d.value = s.value))`
 		const onConflict = opts.overwrite
 			? 'excluded.data'
-			: 'json_patch(excluded.data, player_settings.data)'
+			: `json_patch(json_patch(player_settings.data, excluded.data), ${d1OwnKeys})`
 		const statements = rows.map(
 			(r) =>
 				`INSERT INTO player_settings (account_id, data) VALUES (${r.accountId}, '${sqlStr(r.data)}') ON CONFLICT (account_id) DO UPDATE SET data = ${onConflict};`
 		)
 		console.log(
 			`${rows.length} player(s) to import, ${empty} empty, ${skipped.length} skipped` +
-				(opts.overwrite ? ' (KV overwrites existing rows)' : ' (existing rows merge, D1 wins)')
+				(opts.overwrite
+					? ' (KV overwrites existing rows)'
+					: ' (existing rows merge; D1 wins unless still at a seeded default)')
 		)
 		if (opts.dryRun) {
 			console.log(
@@ -561,16 +597,20 @@ const settingsImportKv = new Command('settings-import-kv')
 
 		// Prove it landed: a merge never deletes, so the table holds at least every imported id.
 		const after = await count()
+		// Chunked: one IN list of every id outgrows D1's statement limit (and argv) at ~15k players.
 		const ids = rows.map((r) => r.accountId)
-		const found = Number(
-			(
-				await execSql(
-					`SELECT COUNT(*) AS n FROM player_settings WHERE account_id IN (${ids.join(', ')})`,
-					remote,
-					'playersettings'
-				)
-			).results[0]?.n ?? 0
-		)
+		let found = 0
+		for (let i = 0; i < ids.length; i += 1000) {
+			found += Number(
+				(
+					await execSql(
+						`SELECT COUNT(*) AS n FROM player_settings WHERE account_id IN (${ids.slice(i, i + 1000).join(', ')})`,
+						remote,
+						'playersettings'
+					)
+				).results[0]?.n ?? 0
+			)
+		}
 		if (found !== ids.length) {
 			throw new Error(
 				`import did not land: ${ids.length - found} of ${ids.length} rows are missing. Re-run it.`