mirror of
https://github.com/djdevin/recflare.git
synced 2026-10-27 07:58:55 -07:00
kv-to-d1
This commit is contained in:
Devin Zuczek
@@ -14,7 +14,7 @@ export * from './clubs-db'
|
||||
export * from './d1-binds'
|
||||
export * from './images-db'
|
||||
export * from './password'
|
||||
export * from './player-settings-kv'
|
||||
export * from './player-settings-db'
|
||||
export * from './rooms-db'
|
||||
export * from './room-instance-db'
|
||||
export * from './room-comments-db'
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/**
|
||||
* The player settings map — the `player_settings` table on the shared `recflare` D1.
|
||||
*
|
||||
* One row per player, holding every setting they have as a flat `{ [Key]: Value }` JSON
|
||||
* record of strings in `data` — OOBE state, tutorial mask, chat privacy, the lot. The
|
||||
* `playersettings` worker owns the table (migration: apps/playersettings/migrations/
|
||||
* 0001_player_settings.sql) but `api`, `match` and `chat` each keep a setting of their own in
|
||||
* the same map, so every writer merges into the record rather than replacing it, and all of
|
||||
* them go through here.
|
||||
*
|
||||
* This replaced a Workers KV namespace (`RECFLARE_PLAYER_SETTINGS`, key `player:<id>`): the
|
||||
* row's `data` is byte-for-byte what the KV value was, which is what lets `runx admin
|
||||
* settings-import-kv` copy the old namespace across. The client is noisy — it re-posts
|
||||
* settings it already has at every login and menu change — and every write site has to read
|
||||
* the map first to merge into it, so comparing the result against what was read is free:
|
||||
* {@link putPlayerSettingsIfChanged} skips the write when the merged map is the one already
|
||||
* stored. (D1 reads are strongly consistent, so unlike KV a write here is visible to the next
|
||||
* read from any PoP.)
|
||||
*/
|
||||
|
||||
export type PlayerSettings = Record<string, string>
|
||||
|
||||
/** Schema DDL (mirror of apps/playersettings/migrations/0001_player_settings.sql). */
|
||||
export const PLAYER_SETTINGS_SCHEMA_DDL: string[] = [
|
||||
`CREATE TABLE IF NOT EXISTS player_settings (
|
||||
account_id INTEGER PRIMARY KEY,
|
||||
data TEXT NOT NULL
|
||||
)`,
|
||||
]
|
||||
|
||||
/**
|
||||
* The player's stored settings, or null when they have none. A D1 failure THROWS rather than
|
||||
* reading as "none": a writer that mistook an outage for an empty map would then store its
|
||||
* one key over everything the player had. Read-only callers that would rather degrade wrap
|
||||
* this in their own `.catch`.
|
||||
*/
|
||||
export async function readPlayerSettings(
|
||||
db: D1Database,
|
||||
accountId: number
|
||||
): Promise<PlayerSettings | null> {
|
||||
const row = await db
|
||||
.prepare('SELECT data FROM player_settings WHERE account_id = ?1')
|
||||
.bind(accountId)
|
||||
.first<{ data: string }>()
|
||||
return row === null ? null : (JSON.parse(row.data) as PlayerSettings)
|
||||
}
|
||||
|
||||
/** Store `settings` as the player's whole map, replacing whatever row they had. */
|
||||
export async function writePlayerSettings(
|
||||
db: D1Database,
|
||||
accountId: number,
|
||||
settings: PlayerSettings
|
||||
): Promise<void> {
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO player_settings (account_id, data) VALUES (?1, ?2)
|
||||
ON CONFLICT (account_id) DO UPDATE SET data = excluded.data`
|
||||
)
|
||||
.bind(accountId, JSON.stringify(settings))
|
||||
.run()
|
||||
}
|
||||
|
||||
/** Drop the player's row entirely — their next read is a first read again. */
|
||||
export async function deletePlayerSettings(db: D1Database, accountId: number): Promise<void> {
|
||||
await db.prepare('DELETE FROM player_settings WHERE account_id = ?1').bind(accountId).run()
|
||||
}
|
||||
|
||||
/** Whether two settings maps hold the same keys with the same values. Order is irrelevant. */
|
||||
export function samePlayerSettings(a: PlayerSettings | null, b: PlayerSettings | null): boolean {
|
||||
const left = a ?? {}
|
||||
const right = b ?? {}
|
||||
const keys = Object.keys(left)
|
||||
if (keys.length !== Object.keys(right).length) return false
|
||||
return keys.every((key) => key in right && left[key] === right[key])
|
||||
}
|
||||
|
||||
/**
|
||||
* Store `next` as the player's settings unless it is what `existing` already says, in which
|
||||
* case nothing is written. Returns whether a write happened.
|
||||
*
|
||||
* `existing` is the map the caller read to build `next` from; passing the map it read (not a
|
||||
* fresh read) is the point — the comparison costs no database operation.
|
||||
*/
|
||||
export async function putPlayerSettingsIfChanged(
|
||||
db: D1Database,
|
||||
accountId: number,
|
||||
existing: PlayerSettings | null,
|
||||
next: PlayerSettings
|
||||
): Promise<boolean> {
|
||||
if (existing !== null && samePlayerSettings(existing, next)) return false
|
||||
await writePlayerSettings(db, accountId, next)
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge `patch` into the player's settings, writing only if a value actually changed. The
|
||||
* result carries the merged map (what the player now has) and whether the row was written.
|
||||
*
|
||||
* Read-modify-write here isn't atomic; racing writers means one player toggling two of their
|
||||
* own options in the same instant, which every caller has accepted since before this helper
|
||||
* existed.
|
||||
*/
|
||||
export async function mergePlayerSettings(
|
||||
db: D1Database,
|
||||
accountId: number,
|
||||
patch: PlayerSettings
|
||||
): Promise<{ settings: PlayerSettings; written: boolean }> {
|
||||
const existing = await readPlayerSettings(db, accountId)
|
||||
const settings: PlayerSettings = { ...existing, ...patch }
|
||||
const written = await putPlayerSettingsIfChanged(db, accountId, existing, settings)
|
||||
return { settings, written }
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
/**
|
||||
* The player settings map in the `RECFLARE_PLAYER_SETTINGS` KV namespace.
|
||||
*
|
||||
* One key per player, `player:<accountId>`, holding every setting they have as a flat
|
||||
* `{ [Key]: Value }` record of strings — OOBE state, tutorial mask, chat privacy, the lot.
|
||||
* The `playersettings` worker owns the namespace but `api`, `match` and `chat` each keep a
|
||||
* setting of their own in the same map, so every writer merges into the record rather than
|
||||
* replacing it, and all of them go through here.
|
||||
*
|
||||
* KV writes cost roughly ten times a read and are capped at one per second per key, and the
|
||||
* client is noisy: it re-posts settings it already has at every login and menu change. Every
|
||||
* write site has to read the map first to merge into it, so comparing the result against what
|
||||
* was read is free — {@link putPlayerSettingsIfChanged} skips the `put` when the merged map
|
||||
* is the one already stored.
|
||||
*/
|
||||
|
||||
export type PlayerSettings = Record<string, string>
|
||||
|
||||
/** The KV key holding a player's settings. */
|
||||
export function playerSettingsKey(accountId: number): string {
|
||||
return `player:${accountId}`
|
||||
}
|
||||
|
||||
/**
|
||||
* The player's stored settings, or null when they have none. A KV failure THROWS rather than
|
||||
* reading as "none": a writer that mistook an outage for an empty map would then store its
|
||||
* one key over everything the player had. Read-only callers that would rather degrade wrap
|
||||
* this in their own `.catch`.
|
||||
*/
|
||||
export async function readPlayerSettings(
|
||||
kv: KVNamespace,
|
||||
accountId: number
|
||||
): Promise<PlayerSettings | null> {
|
||||
return kv.get<PlayerSettings>(playerSettingsKey(accountId), 'json')
|
||||
}
|
||||
|
||||
/** Whether two settings maps hold the same keys with the same values. Order is irrelevant. */
|
||||
export function samePlayerSettings(a: PlayerSettings | null, b: PlayerSettings | null): boolean {
|
||||
const left = a ?? {}
|
||||
const right = b ?? {}
|
||||
const keys = Object.keys(left)
|
||||
if (keys.length !== Object.keys(right).length) return false
|
||||
return keys.every((key) => key in right && left[key] === right[key])
|
||||
}
|
||||
|
||||
/**
|
||||
* Store `next` as the player's settings unless it is what `existing` already says, in which
|
||||
* case nothing is written. Returns whether a write happened.
|
||||
*
|
||||
* `existing` is the map the caller read to build `next` from; passing the map it read (not a
|
||||
* fresh read) is the point — the comparison costs no KV operation.
|
||||
*/
|
||||
export async function putPlayerSettingsIfChanged(
|
||||
kv: KVNamespace,
|
||||
accountId: number,
|
||||
existing: PlayerSettings | null,
|
||||
next: PlayerSettings
|
||||
): Promise<boolean> {
|
||||
if (existing !== null && samePlayerSettings(existing, next)) return false
|
||||
await kv.put(playerSettingsKey(accountId), JSON.stringify(next))
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge `patch` into the player's settings, writing only if a value actually changed. The
|
||||
* result carries the merged map (what the player now has) and whether KV was written.
|
||||
*
|
||||
* Read-modify-write on KV isn't atomic; racing writers here means one player toggling two
|
||||
* of their own options in the same instant, which every caller has accepted since before
|
||||
* this helper existed.
|
||||
*/
|
||||
export async function mergePlayerSettings(
|
||||
kv: KVNamespace,
|
||||
accountId: number,
|
||||
patch: PlayerSettings
|
||||
): Promise<{ settings: PlayerSettings; written: boolean }> {
|
||||
const existing = await readPlayerSettings(kv, accountId)
|
||||
const settings: PlayerSettings = { ...existing, ...patch }
|
||||
const written = await putPlayerSettingsIfChanged(kv, accountId, existing, settings)
|
||||
return { settings, written }
|
||||
}
|
||||
@@ -32,6 +32,7 @@ import type { D1ExecResult } from '../d1'
|
||||
* runx admin grant-plus --username alice [--revoke] [--remote]
|
||||
* runx admin reload-plus <amount> [--dry-run] [--remote]
|
||||
* runx admin cai-load [--file <export.json>] [--dry-run] [--remote]
|
||||
* runx admin settings-import-kv [--namespace-id <id>] [--overwrite] [--dry-run] [--remote]
|
||||
*/
|
||||
|
||||
/**
|
||||
@@ -404,6 +405,184 @@ const caiLoad = new Command('cai-load')
|
||||
console.log(chalk.green(`✓ custom avatar items loaded into ${target(remote)}`))
|
||||
})
|
||||
|
||||
/** The KV binding the player settings used to live under — the key `RECFLARE_KV` files its id by. */
|
||||
const SETTINGS_KV_BINDING = 'RECFLARE_PLAYER_SETTINGS'
|
||||
/** Every settings key in that namespace was `player:<accountId>`. */
|
||||
const SETTINGS_KV_PREFIX = 'player:'
|
||||
/** The KV bulk-get API takes at most this many keys per request. */
|
||||
const KV_BULK_GET_LIMIT = 100
|
||||
|
||||
/**
|
||||
* Run `wrangler kv …` from the `playersettings` directory — the namespace was that worker's
|
||||
* binding, so with `--local` the dev-store state wrangler kept for it is under its .wrangler/.
|
||||
*/
|
||||
async function runKv(args: string[], remote: boolean): Promise<string> {
|
||||
cd(path.join(getRepoRoot(), 'apps', 'playersettings'))
|
||||
const out = await $`pnpm exec wrangler kv ${args} ${remote ? '--remote' : '--local'}`.quiet()
|
||||
return out.stdout
|
||||
}
|
||||
|
||||
/** The JSON wrangler prints after its banner (which can carry ANSI `[` sequences of its own). */
|
||||
function parseWranglerJson<T>(stdout: string): T {
|
||||
const m = /^[[{]/m.exec(stdout)
|
||||
if (!m) throw new Error(`unexpected wrangler output:\n${stdout}`)
|
||||
return JSON.parse(stdout.slice(m.index)) as T
|
||||
}
|
||||
|
||||
/** The old namespace's id, from the `RECFLARE_KV` entry the deploys used to splice it from. */
|
||||
async function settingsNamespaceIdFromEnv(): Promise<string> {
|
||||
const raw = await readRootEnv('RECFLARE_KV')
|
||||
const id = raw ? (JSON.parse(raw) as Record<string, string>)[SETTINGS_KV_BINDING] : undefined
|
||||
if (id) return id
|
||||
throw new Error(
|
||||
`no ${SETTINGS_KV_BINDING} id in RECFLARE_KV — pass --namespace-id, or add the old namespace's id to .env`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy the old `RECFLARE_PLAYER_SETTINGS` KV namespace into the `player_settings` table (owned
|
||||
* by the `playersettings` worker). A KV value was the player's whole map as a JSON object of
|
||||
* strings, which is exactly what the row's `data` holds, so each key becomes one row.
|
||||
*
|
||||
* Rows that already exist are MERGED, with D1 winning: `json_patch(kv, d1)` keeps every key
|
||||
* the live row has and adds the ones only KV knew. That makes the import safe to run after
|
||||
* the workers have cut over (a player who has since toggled something keeps it) and to
|
||||
* re-run. `--overwrite` makes KV win outright instead. Nothing is deleted on either side.
|
||||
*/
|
||||
const settingsImportKv = new Command('settings-import-kv')
|
||||
.description('Copy the old RECFLARE_PLAYER_SETTINGS KV namespace into player_settings (merges)')
|
||||
.option(
|
||||
'--namespace-id <id>',
|
||||
`The KV namespace to read (default: ${SETTINGS_KV_BINDING} in RECFLARE_KV from .env; "local" with --local)`
|
||||
)
|
||||
.option(
|
||||
'--overwrite',
|
||||
'Let the KV value replace a row that already exists (the default merges, keeping what D1 has)',
|
||||
false
|
||||
)
|
||||
.option('--dry-run', 'Read the namespace, report what would be written, change nothing', false)
|
||||
.option('--local', 'Target the local dev database and KV store (the default).', false)
|
||||
.option('--remote', 'Target the deployed database and namespace instead.', false)
|
||||
.action(async (opts) => {
|
||||
const remote = resolveRemote(opts)
|
||||
const namespaceId = opts.namespaceId ?? (remote ? await settingsNamespaceIdFromEnv() : 'local')
|
||||
console.log(
|
||||
`Reading KV namespace ${namespaceId} (${remote ? 'remote' : 'local'}) into ${target(remote)}`
|
||||
)
|
||||
|
||||
const keys = parseWranglerJson<Array<{ name: string }>>(
|
||||
await runKv(
|
||||
['key', 'list', '--namespace-id', namespaceId, '--prefix', SETTINGS_KV_PREFIX],
|
||||
remote
|
||||
)
|
||||
).map((k) => k.name)
|
||||
console.log(`${keys.length} ${SETTINGS_KV_PREFIX}* key(s) in the namespace`)
|
||||
|
||||
const rows: Array<{ accountId: number; data: string }> = []
|
||||
const skipped: string[] = []
|
||||
let empty = 0
|
||||
for (let i = 0; i < keys.length; i += KV_BULK_GET_LIMIT) {
|
||||
const chunk = keys.slice(i, i + KV_BULK_GET_LIMIT)
|
||||
const file = path.join(os.tmpdir(), `recflare-kv-keys-${Date.now()}-${i}.json`)
|
||||
await fs.writeFile(file, JSON.stringify(chunk))
|
||||
let got: Record<string, { value: string | null }>
|
||||
try {
|
||||
got = parseWranglerJson(
|
||||
await runKv(['bulk', 'get', file, '--namespace-id', namespaceId], remote)
|
||||
)
|
||||
} finally {
|
||||
await fs.remove(file)
|
||||
}
|
||||
for (const key of chunk) {
|
||||
const accountId = Number(key.slice(SETTINGS_KV_PREFIX.length))
|
||||
const value = got[key]?.value
|
||||
let parsed: unknown = null
|
||||
try {
|
||||
parsed = value == null ? null : JSON.parse(value)
|
||||
} catch {
|
||||
parsed = null
|
||||
}
|
||||
const isMap =
|
||||
parsed !== null &&
|
||||
typeof parsed === 'object' &&
|
||||
!Array.isArray(parsed) &&
|
||||
Object.values(parsed).every((v) => typeof v === 'string')
|
||||
if (!Number.isSafeInteger(accountId) || accountId <= 0 || !isMap) {
|
||||
skipped.push(key)
|
||||
continue
|
||||
}
|
||||
// An empty map is what a player with no row reads as; nothing to import.
|
||||
if (Object.keys(parsed as object).length === 0) {
|
||||
empty++
|
||||
continue
|
||||
}
|
||||
rows.push({ accountId, data: JSON.stringify(parsed) })
|
||||
}
|
||||
console.log(` fetched ${Math.min(i + chunk.length, keys.length)}/${keys.length}`)
|
||||
}
|
||||
for (const key of skipped) console.log(chalk.yellow(` skipping ${key}: not a settings map`))
|
||||
|
||||
const onConflict = opts.overwrite
|
||||
? 'excluded.data'
|
||||
: 'json_patch(excluded.data, player_settings.data)'
|
||||
const statements = rows.map(
|
||||
(r) =>
|
||||
`INSERT INTO player_settings (account_id, data) VALUES (${r.accountId}, '${sqlStr(r.data)}') ON CONFLICT (account_id) DO UPDATE SET data = ${onConflict};`
|
||||
)
|
||||
console.log(
|
||||
`${rows.length} player(s) to import, ${empty} empty, ${skipped.length} skipped` +
|
||||
(opts.overwrite ? ' (KV overwrites existing rows)' : ' (existing rows merge, D1 wins)')
|
||||
)
|
||||
if (opts.dryRun) {
|
||||
console.log(
|
||||
chalk.cyan(`--dry-run: built ${statements.length} statements; nothing was written.`)
|
||||
)
|
||||
return
|
||||
}
|
||||
if (rows.length === 0) {
|
||||
console.log(chalk.green('✓ nothing to import'))
|
||||
return
|
||||
}
|
||||
|
||||
const count = async (): Promise<number> =>
|
||||
Number(
|
||||
(await execSql('SELECT COUNT(*) AS n FROM player_settings', remote, 'playersettings'))
|
||||
.results[0]?.n ?? 0
|
||||
)
|
||||
const before = await count()
|
||||
|
||||
const sqlFile = path.join(os.tmpdir(), `recflare-settings-${Date.now()}.sql`)
|
||||
await fs.writeFile(sqlFile, statements.join('\n'))
|
||||
try {
|
||||
await execSqlFile(sqlFile, remote, 'playersettings')
|
||||
} finally {
|
||||
await fs.remove(sqlFile)
|
||||
}
|
||||
|
||||
// Prove it landed: a merge never deletes, so the table holds at least every imported id.
|
||||
const after = await count()
|
||||
const ids = rows.map((r) => r.accountId)
|
||||
const found = Number(
|
||||
(
|
||||
await execSql(
|
||||
`SELECT COUNT(*) AS n FROM player_settings WHERE account_id IN (${ids.join(', ')})`,
|
||||
remote,
|
||||
'playersettings'
|
||||
)
|
||||
).results[0]?.n ?? 0
|
||||
)
|
||||
if (found !== ids.length) {
|
||||
throw new Error(
|
||||
`import did not land: ${ids.length - found} of ${ids.length} rows are missing. Re-run it.`
|
||||
)
|
||||
}
|
||||
const added = after - before
|
||||
console.log(
|
||||
`${added} new row(s), ${rows.length - added} merged into existing rows (${after} rows now)`
|
||||
)
|
||||
console.log(chalk.green(`✓ player settings imported into ${target(remote)}`))
|
||||
})
|
||||
|
||||
const lookup = new Command('lookup')
|
||||
.description('Print an account by id or username')
|
||||
.option('--account <id>', 'Account id to look up')
|
||||
@@ -467,6 +646,7 @@ export const adminCmd = new Command('admin')
|
||||
.addCommand(grantStudio)
|
||||
.addCommand(reloadPlus)
|
||||
.addCommand(caiLoad)
|
||||
.addCommand(settingsImportKv)
|
||||
.addCommand(lookup)
|
||||
.addHelpText(
|
||||
'after',
|
||||
|
||||
Reference in New Issue
Block a user