diff --git a/.env.example b/.env.example index fc01e2a..016ee2f 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,27 @@ RECFLARE_DOMAIN=rec.example.com # spliced in at deploy time. Required to deploy any worker. # RECFLARE_SECRETS_STORE=00000000-0000-0000-0000-000000000000 +# OPTIONAL: run the workers on a libSQL server (Turso) INSTEAD of the D1 database above. Both +# are SQLite; the schema, the migrations and the workers are the same, and the D1 database +# stays bound (and RECFLARE_D1 still required) so switching back is unsetting this line and +# redeploying. Leave both unset to use D1, which is the default. +# +# The URL is the one Turso shows for the database (`turso db show --url`), as-is: +# +# RECFLARE_LIBSQL_DB_URL=libsql://-.turso.io +# +# The token (`turso db tokens create `) is the one entry in this file that IS a secret. +# It is never passed as a var: `just deploy` puts it as an encrypted worker secret, and +# `just migrate` / `runx admin --remote` read it from here to reach the database directly. +# +# RECFLARE_LIBSQL_DB_AUTH_TOKEN=eyJ... +# +# `just dev` and the tests ignore both and stay on the local D1. Setting the URL changes what +# `just migrate` and every `--remote` operator command talk to — the libSQL database, not D1 — +# so it is one switch for the whole deployment: the workers, their migrations and the tooling +# all move together, which is the only sane state for fifteen workers sharing one database. +# To carry existing data over, see "Using Turso instead of D1" in DEPLOYING.md. + # --- Server tuning (all optional; the shown value is the built-in default) --- # Everything below is passed to the workers as a variable, named without the RECFLARE_ # prefix: RECFLARE_STARTING_TOKENS becomes STARTING_TOKENS. Every worker gets every knob — diff --git a/CLAUDE.md b/CLAUDE.md index db431f6..5fc520f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,6 +37,10 @@ - Syncpack ensures dependency version consistency - Turborepo enables parallel task execution and caching - Workers configured via `wrangler.jsonc` with environment variables +- The shared `recflare` database is D1 by default, or a libSQL server (Turso) when + `LIBSQL_DB_URL` is set: `withDatabase` (`packages/domain/src/libsql-db.ts`) wraps each + worker's default export and swaps `env.DB` for an adapter with the D1 API, so query code + only ever sees `D1Database`. Migrations and `runx` tooling follow the same switch. - Each worker has `context.ts` for typed environment bindings - Integration tests in `src/test/integration/` - Workers use `nodejs_compat` compatibility flag diff --git a/DEPLOYING.md b/DEPLOYING.md index 80217e4..ef001aa 100644 --- a/DEPLOYING.md +++ b/DEPLOYING.md @@ -147,6 +147,42 @@ just migrate # migrate every worker that owns migrations just migrate -F rooms # or scope to one worker ``` +### Using Turso instead of D1 + +The workers can run on a [Turso](https://turso.tech) (libSQL) database instead of D1, +with nothing else changing: same schema, same migrations, same workers. D1 stays the +default and stays bound, so this is a switch, not a migration of the code. The +switch is two lines in `.env`: + +```bash +turso db create recflare +turso db show recflare --url # -> RECFLARE_LIBSQL_DB_URL=libsql://... +turso db tokens create recflare # -> RECFLARE_LIBSQL_DB_AUTH_TOKEN=... +``` + +With both set, `just migrate` applies the workers' migrations to that database (with +the same per-worker history tables wrangler keeps on D1), `just deploy` hands every +worker the URL as a var and the token as an encrypted worker secret, and the +`--remote` operator commands (`just admin`, `just catalog`) act on it. Local dev and +the tests keep using the local D1 regardless. + +To carry an existing D1 database across, export it and load the dump before the +first `just migrate` — the dump includes the migration history, so nothing is +re-applied: + +```bash +wrangler d1 export recflare --remote --output recflare.sql # from any D1-backed app dir +turso db shell recflare < recflare.sql +``` + +To go back to D1, unset (or comment out) the two lines and run `just deploy`. The +token secret is left on the workers and ignored without a URL; remove it with +`wrangler secret delete LIBSQL_DB_AUTH_TOKEN` per worker if you want it gone. + +> ⚠️ The URL is one switch for the whole deployment. Deploy every worker after +> changing it: a worker left on the old database would be reading and writing +> different data from the rest. + ### R2 and Durable Objects You only have to create the buckets: diff --git a/apps/accounts/src/accounts.app.ts b/apps/accounts/src/accounts.app.ts index d351336..b6231f1 100644 --- a/apps/accounts/src/accounts.app.ts +++ b/apps/accounts/src/accounts.app.ts @@ -11,6 +11,7 @@ import { getAccountsByIds, searchAccounts, updateAccount, + withDatabase, } from '@repo/domain' import { logger, @@ -55,7 +56,7 @@ import { resolveWhitelistedEmoji, WHITELISTED_EMOJIS } from './whitelisted-emoji import type { Context } from 'hono' import type { DescribeRouteOptions } from 'hono-openapi' import type { Account } from '@repo/domain' -import type { App } from './context' +import type { App, Env } from './context' /** * Account reads/writes are backed by the shared `accounts` table in D1 (schema @@ -856,4 +857,4 @@ app.get( ) ) -export default app +export default withDatabase(app) diff --git a/apps/api/src/api.app.ts b/apps/api/src/api.app.ts index a209cee..1c919e3 100644 --- a/apps/api/src/api.app.ts +++ b/apps/api/src/api.app.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono' import { describeRoute, openAPIRouteHandler } from 'hono-openapi' import { useWorkersLogger } from 'workers-tagged-logger' +import { withDatabase } from '@repo/domain' import { withCleanSpec, withDefaultCors, withNotFound, withOnError } from '@repo/hono-helpers' import { accountRoutes } from './routes/account' @@ -16,7 +17,7 @@ import { progressionRoutes } from './routes/progression' import { roomRoutes } from './routes/rooms' import { socialRoutes } from './routes/social' -import type { App } from './context' +import type { App, Env } from './context' /** * The Game API surface. Endpoints that would be backed by a database or on-disk @@ -105,4 +106,4 @@ app.get( ) ) -export default app +export default withDatabase(app) diff --git a/apps/auth/src/auth.app.ts b/apps/auth/src/auth.app.ts index 5ff53d4..4a0ff35 100644 --- a/apps/auth/src/auth.app.ts +++ b/apps/auth/src/auth.app.ts @@ -21,6 +21,7 @@ import { subRoomDataBlob, updateAccount, verifyPassword, + withDatabase, writeAuditLog, } from '@repo/domain' import { @@ -65,7 +66,7 @@ import { verifySteamTicket } from './steam-ticket' import type { Context } from 'hono' import type { Account } from '@repo/domain' -import type { App } from './context' +import type { App, Env } from './context' import type { PlatformLink } from './platform-db' /** OAuth scopes granted by `/connect/token`. */ @@ -1351,4 +1352,4 @@ app.get( ) ) -export default app +export default withDatabase(app) diff --git a/apps/chat/package.json b/apps/chat/package.json index 2fd64e5..952bf06 100644 --- a/apps/chat/package.json +++ b/apps/chat/package.json @@ -17,6 +17,7 @@ }, "dependencies": { "@2toad/profanity": "3.3.0", + "@repo/domain": "workspace:*", "@repo/hono-helpers": "workspace:*", "@repo/jwt": "workspace:*", "@standard-community/standard-json": "0.3.5", diff --git a/apps/chat/src/chat.app.ts b/apps/chat/src/chat.app.ts index c96683d..f3b3258 100644 --- a/apps/chat/src/chat.app.ts +++ b/apps/chat/src/chat.app.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono' import { describeRoute, openAPIRouteHandler } from 'hono-openapi' import { useWorkersLogger } from 'workers-tagged-logger' +import { withDatabase } from '@repo/domain' import { logger, withCleanSpec, withNotFound, withOnError } from '@repo/hono-helpers' import { validateAndGetAccountId } from '@repo/jwt' @@ -40,9 +41,9 @@ import { ChatThreadType, createThread, getOrCreateThreadWithMembers, + getPartyThreadForPlayer, getThreadForPlayer, getThreadMemberIds, - getPartyThreadForPlayer, getThreadMeta, getThreadsForPlayer, isThreadMember, @@ -1557,4 +1558,4 @@ app.get( ) ) -export default app +export default withDatabase(app) diff --git a/apps/clubs/src/clubs.app.ts b/apps/clubs/src/clubs.app.ts index 1cf7f3a..aef213f 100644 --- a/apps/clubs/src/clubs.app.ts +++ b/apps/clubs/src/clubs.app.ts @@ -36,6 +36,7 @@ import { setMemberType, subscribeToClub, updateClub, + withDatabase, } from '@repo/domain' import { intVar, logger, withCleanSpec, withNotFound, withOnError } from '@repo/hono-helpers' import { validateAndGetAccountId } from '@repo/jwt' @@ -76,7 +77,7 @@ import { } from './openapi' import type { Context } from 'hono' -import type { App } from './context' +import type { App, Env } from './context' /** * Clubs Worker. Hosts the club endpoints the game client calls on the `clubs` host: @@ -1706,4 +1707,4 @@ app.get( ) ) -export default app +export default withDatabase(app) diff --git a/apps/econ/src/econ.app.ts b/apps/econ/src/econ.app.ts index 46f2b6d..b3a885a 100644 --- a/apps/econ/src/econ.app.ts +++ b/apps/econ/src/econ.app.ts @@ -17,6 +17,7 @@ import { levelsReached, ownsInvention, setOutfit, + withDatabase, } from '@repo/domain' import { intVar, logger, withCleanSpec, withNotFound, withOnError } from '@repo/hono-helpers' import { validateAndGetAccountId, validateAndGetPlus, validateAndGetVersion } from '@repo/jwt' @@ -5918,4 +5919,4 @@ export const scheduled: ExportedHandlerScheduledHandler = (_controller, env // A Worker only runs `scheduled` when it's on the default export, so econ takes the object // form the runtime requires (as `www` does for its role sweep and `match` for presence). -export default { fetch: app.fetch, scheduled } satisfies ExportedHandler +export default withDatabase({ fetch: app.fetch, scheduled }) diff --git a/apps/img/package.json b/apps/img/package.json index 1a07dd8..3d83a12 100644 --- a/apps/img/package.json +++ b/apps/img/package.json @@ -17,6 +17,7 @@ }, "dependencies": { "@cf-wasm/photon": "^0.3.6", + "@repo/domain": "workspace:*", "@repo/hono-helpers": "workspace:*", "@standard-community/standard-json": "0.3.5", "@standard-community/standard-openapi": "0.2.9", diff --git a/apps/img/src/img.app.ts b/apps/img/src/img.app.ts index 7d9cc6b..5f7b1e2 100644 --- a/apps/img/src/img.app.ts +++ b/apps/img/src/img.app.ts @@ -3,6 +3,7 @@ import { Hono } from 'hono' import { describeRoute, openAPIRouteHandler } from 'hono-openapi' import { useWorkersLogger } from 'workers-tagged-logger' +import { withDatabase } from '@repo/domain' import { withCleanSpec, withNotFound, withOnError, writeContentRange } from '@repo/hono-helpers' import { imageBytes, json, ServiceStatus } from './openapi' @@ -562,4 +563,4 @@ app.get( } ) -export default app +export default withDatabase(app) diff --git a/apps/leaderboard/src/leaderboard.app.ts b/apps/leaderboard/src/leaderboard.app.ts index 53c56ed..8dafe61 100644 --- a/apps/leaderboard/src/leaderboard.app.ts +++ b/apps/leaderboard/src/leaderboard.app.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono' import { describeRoute, openAPIRouteHandler } from 'hono-openapi' import { useWorkersLogger } from 'workers-tagged-logger' +import { withDatabase } from '@repo/domain' import { logger, withCleanSpec, withNotFound, withOnError } from '@repo/hono-helpers' import { validateAndGetAccountId } from '@repo/jwt' @@ -14,7 +15,6 @@ import { NO_SCORE, UNRANKED, } from './leaderboard-db' - import { CheckAndSetStatBody, CheckAndSetStatResponse, @@ -27,7 +27,7 @@ import { PlayerRank, } from './openapi' -import type { App } from './context' +import type { App, Env } from './context' import type { Board } from './leaderboard-db' /** @@ -73,7 +73,8 @@ function board(body: BoardBody): Board { return { roomId: int(body.RoomId, 0), statChannel: int(body.StatChannel, 0), - ...(int(body.FilterType, 0) === FilterType.Friends && playerId !== 0 && { friendsOf: playerId }), + ...(int(body.FilterType, 0) === FilterType.Friends && + playerId !== 0 && { friendsOf: playerId }), } } @@ -353,4 +354,4 @@ app.get( ) ) -export default app +export default withDatabase(app) diff --git a/apps/lists/src/lists.app.ts b/apps/lists/src/lists.app.ts index 86aed7e..8917bb2 100644 --- a/apps/lists/src/lists.app.ts +++ b/apps/lists/src/lists.app.ts @@ -10,6 +10,7 @@ import { getPlayerList, getRecentlyUpdatedRooms, getVisitedRooms, + withDatabase, } from '@repo/domain' import { withCleanSpec, withNotFound, withOnError } from '@repo/hono-helpers' import { validateAndGetAccountId } from '@repo/jwt' @@ -41,7 +42,7 @@ import { import type { Context } from 'hono' import type { CuratedList, Room } from '@repo/domain' import type { CatalogKindValue } from '../../econ/src/catalog-load' -import type { App } from './context' +import type { App, Env } from './context' // The item catalog's sellable-rarity rule, from the worker that owns the table. Imported // rather than restated so a row can never offer an id the storefront omits. @@ -1006,4 +1007,4 @@ app.get( ) ) -export default app +export default withDatabase(app) diff --git a/apps/match/src/match.app.ts b/apps/match/src/match.app.ts index c6ad910..938d835 100644 --- a/apps/match/src/match.app.ts +++ b/apps/match/src/match.app.ts @@ -45,6 +45,7 @@ import { setRoomInstanceInProgress, setRoomInstancePrivate, subRoomDataBlob, + withDatabase, } from '@repo/domain' import { logger, withCleanSpec, withNotFound, withOnError } from '@repo/hono-helpers' import { generatePhotonAuthToken, validateAndGetAccountId, validateAndGetVersion } from '@repo/jwt' @@ -3365,4 +3366,4 @@ export const scheduled: ExportedHandlerScheduledHandler = (_controller, env // Standalone entry: a Worker only runs `scheduled` when it's on the default export, // so match keeps the object form the runtime requires to fire its `*/5 * * * *` cron. -export default { fetch: app.fetch, scheduled } satisfies ExportedHandler +export default withDatabase({ fetch: app.fetch, scheduled }) diff --git a/apps/mono/src/mono.app.ts b/apps/mono/src/mono.app.ts index addc280..b903e38 100644 --- a/apps/mono/src/mono.app.ts +++ b/apps/mono/src/mono.app.ts @@ -20,6 +20,8 @@ * Cloudflare allows only one static-assets binding per Worker. Resolve that (serve * their static trees from R2, or keep those three as their own Workers) before adding. */ +import { withDatabase } from '@repo/domain' + import accounts from '../../accounts/src/accounts.app' import api from '../../api/src/api.app' import auth from '../../auth/src/auth.app' @@ -45,24 +47,34 @@ type Mounted = { fetch(request: Request, env: Env, ctx: ExecutionContext): Response | Promise } +/** + * Mount an app on this worker's Env. Each app declares only the bindings it reads, and + * this Env is the union of them, so the app's narrower Env is not checked against this one + * here — as it never was through Hono's `fetch`, which accepts any env. `withDatabase` + * (@repo/domain) types a wrapped app's `fetch` by its own Env, which is what needs the cast. + */ +const mount = (app: { + fetch(request: Request, env: never, ctx: ExecutionContext): Response | Promise +}): Mounted => app as Mounted + /** * Subdomain -> mounted app. Keys must match the `` in `.` from the * `ns` service-discovery document so production host-based routing lines up. */ const services = { - accounts, - api, - auth, - cdn, - chat, - clubs, - commerce, - match, - notify, - ns, - playersettings, - rooms, - storage, + accounts: mount(accounts), + api: mount(api), + auth: mount(auth), + cdn: mount(cdn), + chat: mount(chat), + clubs: mount(clubs), + commerce: mount(commerce), + match: mount(match), + notify: mount(notify), + ns: mount(ns), + playersettings: mount(playersettings), + rooms: mount(rooms), + storage: mount(storage), } satisfies Record type ServiceName = keyof typeof services @@ -90,7 +102,7 @@ function resolve(request: Request): { name: ServiceName; request: Request } | un return undefined } -export default { +export default withDatabase({ fetch(request: Request, env: Env, ctx: ExecutionContext): Response | Promise { const resolved = resolve(request) if (resolved === undefined) { @@ -117,4 +129,4 @@ export default { matchScheduled(controller, env, ctx) roomsScheduled(controller, env, ctx) }, -} satisfies ExportedHandler +}) diff --git a/apps/notify/src/notify.app.ts b/apps/notify/src/notify.app.ts index 7ab960b..0059c1e 100644 --- a/apps/notify/src/notify.app.ts +++ b/apps/notify/src/notify.app.ts @@ -1,7 +1,7 @@ import { Hono } from 'hono' import { useWorkersLogger } from 'workers-tagged-logger' -import { createNotification, writeAuditLog } from '@repo/domain' +import { createNotification, withDatabase, writeAuditLog } from '@repo/domain' import { logger, withDefaultCors, withNotFound, withOnError } from '@repo/hono-helpers' import { validateAndGetAccountId, validateAndGetRoles } from '@repo/jwt' @@ -13,7 +13,7 @@ import { } from './notifications-hub' import type { Context, MiddlewareHandler } from 'hono' -import type { App } from './context' +import type { App, Env } from './context' /** * Maps a SignalR hub at `/hub/v1`. The hub itself — WebSocket transport, the @@ -371,4 +371,4 @@ const app = new Hono() }) export { NotificationsHub } -export default app +export default withDatabase(app) diff --git a/apps/roomcomments/src/roomcomments.app.ts b/apps/roomcomments/src/roomcomments.app.ts index d326f04..298061b 100644 --- a/apps/roomcomments/src/roomcomments.app.ts +++ b/apps/roomcomments/src/roomcomments.app.ts @@ -2,7 +2,12 @@ import { Hono } from 'hono' import { describeRoute, openAPIRouteHandler } from 'hono-openapi' import { useWorkersLogger } from 'workers-tagged-logger' -import { createRoomComment, DEFAULT_COMMENT_COUNT, getRoomComments } from '@repo/domain' +import { + createRoomComment, + DEFAULT_COMMENT_COUNT, + getRoomComments, + withDatabase, +} from '@repo/domain' import { withCleanSpec, withNotFound, withOnError } from '@repo/hono-helpers' import { validateAndGetAccountId } from '@repo/jwt' @@ -17,7 +22,7 @@ import { } from './openapi' import type { Context } from 'hono' -import type { App } from './context' +import type { App, Env } from './context' /** * Room Comments Worker. Serves the notes a player pins in a room's scene — a message, a @@ -257,4 +262,4 @@ app.get( ) ) -export default app +export default withDatabase(app) diff --git a/apps/rooms/src/rooms.app.ts b/apps/rooms/src/rooms.app.ts index f0afd03..275df6d 100644 --- a/apps/rooms/src/rooms.app.ts +++ b/apps/rooms/src/rooms.app.ts @@ -74,6 +74,7 @@ import { transferRoomOwnership, unbanPlayerFromRoom, updateRoomFields, + withDatabase, writeAuditLog, } from '@repo/domain' import { @@ -4455,4 +4456,4 @@ export const scheduled: ExportedHandlerScheduledHandler = (_controller, env // Standalone entry: a Worker only runs `scheduled` when it's on the default export, // so rooms keeps the object form the runtime requires to fire its `*/5 * * * *` cron. -export default { fetch: app.fetch, scheduled } satisfies ExportedHandler +export default withDatabase({ fetch: app.fetch, scheduled }) diff --git a/apps/www/src/www.app.ts b/apps/www/src/www.app.ts index 17756d6..7ac5dab 100644 --- a/apps/www/src/www.app.ts +++ b/apps/www/src/www.app.ts @@ -1,6 +1,7 @@ import { Hono } from 'hono' import { useWorkersLogger } from 'workers-tagged-logger' +import { withDatabase } from '@repo/domain' import { getAccount, updateAccount } from '@repo/domain/src/accounts-db' import { PlatformType } from '@repo/domain/src/enums' import { countOnlinePlayers } from '@repo/domain/src/presence-db' @@ -539,4 +540,4 @@ export const scheduled: ExportedHandlerScheduledHandler = (_controller, env // A Worker only runs `scheduled` when it's on the default export, so www takes the object // form the runtime requires (as `match` does for its presence sweep). -export default { fetch: app.fetch, scheduled } satisfies ExportedHandler +export default withDatabase({ fetch: app.fetch, scheduled }) diff --git a/packages/domain/package.json b/packages/domain/package.json index 1a71706..edcfd18 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -7,14 +7,17 @@ "main": "src/index.ts", "scripts": { "check:lint": "run-oxlint", - "check:types": "run-tsc" + "check:types": "run-tsc", + "test": "run-vitest" + }, + "dependencies": { + "@libsql/client": "0.18.0", + "isemail": "^3.2.0" }, "devDependencies": { "@cloudflare/workers-types": "4.20260630.1", "@repo/tools": "workspace:*", - "@repo/typescript-config": "workspace:*" - }, - "dependencies": { - "isemail": "^3.2.0" + "@repo/typescript-config": "workspace:*", + "vitest": "4.1.9" } } diff --git a/packages/domain/src/index.ts b/packages/domain/src/index.ts index 3d3cefb..8e0fa21 100644 --- a/packages/domain/src/index.ts +++ b/packages/domain/src/index.ts @@ -10,6 +10,7 @@ export * from './accounts-db' export * from './audit-db' export * from './clubs-db' export * from './d1-binds' +export * from './libsql-db' export * from './images-db' export * from './password' export * from './rooms-db' diff --git a/packages/domain/src/libsql-db.test.ts b/packages/domain/src/libsql-db.test.ts new file mode 100644 index 0000000..0e42796 --- /dev/null +++ b/packages/domain/src/libsql-db.test.ts @@ -0,0 +1,175 @@ +import { createClient } from '@libsql/client' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { bindPlaceholders, chunkForBinds } from './d1-binds' +import { libsqlDatabase, resolveDatabaseEnv, withDatabase } from './libsql-db' + +import type { Client } from '@libsql/client' + +/** + * The adapter against a real (in-memory) libSQL database, asserting the D1 behaviours the + * workers lean on: `first` is null on no row, `run`/`all` carry `results` and + * `meta.changes`, `batch` is one transaction, numbered placeholders bind by position. + */ +describe('libsqlDatabase', () => { + let client: Client + let db: D1Database + + beforeEach(async () => { + client = createClient({ url: ':memory:' }) + db = libsqlDatabase(client) + await db.exec(` + CREATE TABLE account ( + data TEXT NOT NULL, + account_id INTEGER GENERATED ALWAYS AS (json_extract(data, '$.accountId')) VIRTUAL, + username_lower TEXT GENERATED ALWAYS AS (lower(json_extract(data, '$.username'))) VIRTUAL + ); + CREATE UNIQUE INDEX idx_account_id ON account (account_id); + `) + }) + + afterEach(() => { + client.close() + }) + + const insert = (id: number, username: string) => + db + .prepare('INSERT INTO account (data) VALUES (?1)') + .bind(JSON.stringify({ accountId: id, username })) + .run() + + it('runs writes and reports changes like D1', async () => { + const res = await insert(1, 'Alice') + expect(res.success).toBe(true) + expect(res.results).toEqual([]) + expect(res.meta.changes).toBe(1) + expect(res.meta.last_row_id).toBe(1) + + const del = await db.prepare('DELETE FROM account WHERE account_id = ?1').bind(99).run() + expect(del.meta.changes).toBe(0) + }) + + it('reads rows as plain objects, and null when there is none', async () => { + await insert(1, 'Alice') + const row = await db + .prepare( + 'SELECT account_id AS id, username_lower AS name, data FROM account WHERE account_id = ?1' + ) + .bind(1) + .first<{ id: number; name: string; data: string }>() + expect(row).toEqual({ id: 1, name: 'alice', data: '{"accountId":1,"username":"Alice"}' }) + // A plain object, not the client's row shape: no hidden length, spreads cleanly. + expect(Object.keys(row!)).toEqual(['id', 'name', 'data']) + expect({ ...row }).toEqual(row) + + expect( + await db.prepare('SELECT 1 FROM account WHERE account_id = ?1').bind(2).first() + ).toBeNull() + expect(await db.prepare('SELECT account_id FROM account').first('account_id')).toBe(1) + }) + + it('binds numbered placeholders by position, as d1-binds builds them', async () => { + await insert(1, 'a') + await insert(2, 'b') + await insert(3, 'c') + const ids = [3, 1, 42] + const [chunk] = chunkForBinds(ids, 1) + const { results } = await db + .prepare( + `SELECT account_id AS id FROM account WHERE username_lower != ?1 AND account_id IN (${bindPlaceholders(chunk!, 1)}) ORDER BY id` + ) + .bind('zzz', ...chunk!) + .all<{ id: number }>() + expect(results).toEqual([{ id: 1 }, { id: 3 }]) + }) + + it('supports RETURNING through first() the way the auth refresh path uses it', async () => { + await insert(7, 'Seven') + const row = await db + .prepare('DELETE FROM account WHERE account_id = ?1 RETURNING account_id AS accountId') + .bind(7) + .first<{ accountId: number }>() + expect(row).toEqual({ accountId: 7 }) + expect(await db.prepare('SELECT COUNT(*) AS n FROM account').first('n')).toBe(0) + }) + + it('runs a batch as one transaction and returns a result per statement', async () => { + const [a, b, c] = await db.batch<{ n: number }>([ + db.prepare('INSERT INTO account (data) VALUES (?1)').bind('{"accountId":1,"username":"a"}'), + db.prepare('INSERT INTO account (data) VALUES (?1)').bind('{"accountId":2,"username":"b"}'), + db.prepare('SELECT COUNT(*) AS n FROM account'), + ]) + expect(a!.meta.changes).toBe(1) + expect(b!.meta.changes).toBe(1) + expect(c!.results).toEqual([{ n: 2 }]) + + // The second insert violates the unique index; the first must not survive either. + await expect( + db.batch([ + db.prepare('INSERT INTO account (data) VALUES (?1)').bind('{"accountId":3,"username":"c"}'), + db.prepare('INSERT INTO account (data) VALUES (?1)').bind('{"accountId":3,"username":"d"}'), + ]) + ).rejects.toThrow() + expect(await db.prepare('SELECT COUNT(*) AS n FROM account').first('n')).toBe(2) + }) + + it('round-trips blobs and booleans', async () => { + await db.exec('CREATE TABLE blob (id INTEGER PRIMARY KEY, body BLOB, flag INTEGER)') + const bytes = new Uint8Array([1, 2, 3]).buffer + await db + .prepare('INSERT INTO blob (id, body, flag) VALUES (?1, ?2, ?3)') + .bind(1, bytes, true) + .run() + const row = await db.prepare('SELECT body, flag FROM blob WHERE id = ?1').bind(1).first<{ + body: ArrayBuffer + flag: number + }>() + expect(new Uint8Array(row!.body)).toEqual(new Uint8Array([1, 2, 3])) + expect(row!.flag).toBe(1) + }) + + it('refuses an undefined bind, as D1 does', () => { + expect(() => db.prepare('SELECT ?1').bind(undefined)).toThrow(/undefined/) + }) + + it('serves raw() rows as arrays', async () => { + await insert(1, 'Alice') + expect(await db.prepare('SELECT account_id, username_lower FROM account').raw()).toEqual([ + [1, 'alice'], + ]) + expect( + await db.prepare('SELECT account_id, username_lower FROM account').raw({ columnNames: true }) + ).toEqual([ + ['account_id', 'username_lower'], + [1, 'alice'], + ]) + }) +}) + +describe('withDatabase', () => { + const d1 = { tag: 'd1' } as unknown as D1Database + + it('passes the env through untouched when no libSQL URL is set', async () => { + const env = { DB: d1, NAME: 'x' } + let seen: unknown + const handler = withDatabase({ + fetch(_request, e) { + seen = e + return new Response('ok') + }, + }) + await handler.fetch!(new Request('https://example.com'), env, {} as ExecutionContext) + expect(seen).toBe(env) + }) + + it('swaps DB for the libSQL adapter when the URL is set, and only once', () => { + const env = { DB: d1, LIBSQL_DB_URL: 'libsql://db-org.turso.io', LIBSQL_DB_AUTH_TOKEN: 't' } + const resolved = resolveDatabaseEnv(env) + expect(resolved).not.toBe(env) + expect(resolved.DB).not.toBe(d1) + expect(typeof resolved.DB.prepare).toBe('function') + expect(resolved.LIBSQL_DB_URL).toBe(env.LIBSQL_DB_URL) + // A nested handler (mono mounting a worker) sees an env this module already built. + expect(resolveDatabaseEnv(resolved)).toBe(resolved) + }) +}) diff --git a/packages/domain/src/libsql-db.ts b/packages/domain/src/libsql-db.ts new file mode 100644 index 0000000..01c5ffa --- /dev/null +++ b/packages/domain/src/libsql-db.ts @@ -0,0 +1,259 @@ +/** + * The shared `recflare` database on a libSQL server (Turso) instead of Cloudflare D1. + * + * D1 is the default and stays fully supported: every worker still binds it as `DB`, local + * dev and vitest run on miniflare's D1, and an operator who never sets `LIBSQL_DB_URL` + * never loads this. Set the URL (and its auth token) and the SAME workers read and write + * a libSQL database instead, through an adapter that presents the D1 API — `prepare`, + * `bind`, `first`, `all`, `run`, `batch` — over `@libsql/client`. Nothing that queries the + * database knows which backend it is on; the 400-odd `db: D1Database` signatures and the + * hand-written SQL behind them are untouched, which is the point of doing it this way. + * + * Both are SQLite, so the schema, the migrations and the SQL carry across as written + * (generated columns, JSON1, `RETURNING`, numbered `?1` placeholders). Two differences + * worth knowing: + * - D1 caps a statement at 100 bound parameters and the code chunks for that + * (`d1-binds.ts`); libSQL's limit is far higher, so the chunking is merely harmless. + * - A D1 `batch` and a libSQL `batch` are both one transaction, all-or-nothing, so the + * call sites that rely on that (`rooms-db.ts`) behave the same. + * + * The import is `@libsql/client/web`, which the Cloudflare tutorial insists on: the + * default entry is the Node build and does not run in workerd. In the web build a + * `libsql://` URL is Hrana over HTTP — stateless, one fetch per statement, no socket held + * between requests — which is the right transport for a Worker, and why the client is + * created per request rather than cached on the isolate. + * + * Migrations go through `runx libsql migrate` (see packages/tools), which `just migrate` + * switches to on its own when the URL is configured; operator tooling (`runx admin`, + * `runx catalog`) follows the same switch in packages/tools/src/d1.ts. + */ + +import { createClient } from '@libsql/client/web' + +import type { Client, InStatement, InValue, ResultSet } from '@libsql/client/web' + +/** The bindings and vars the switch reads. Every D1-backed worker's Env satisfies this. */ +export interface DatabaseEnv { + /** The D1 binding — used as-is when no libSQL URL is configured. */ + DB: D1Database + /** + * A libSQL server to use INSTEAD of `DB`, e.g. `libsql://-.turso.io`. Set from + * `RECFLARE_LIBSQL_DB_URL` in .env at deploy time (see .env.example). Unset — the usual + * case, and always in local dev and tests — means D1. + */ + LIBSQL_DB_URL?: string + /** + * The server's auth token. Arrives as an encrypted worker secret, which `just deploy` + * sets from `RECFLARE_LIBSQL_DB_AUTH_TOKEN`; never as a plain var. + */ + LIBSQL_DB_AUTH_TOKEN?: string +} + +/** + * The database a request should use: the libSQL adapter when `LIBSQL_DB_URL` is set, the + * D1 binding otherwise. Cheap either way — creating the libSQL client does no I/O. + */ +export function openDatabase(env: DatabaseEnv): D1Database { + const url = env.LIBSQL_DB_URL?.trim() + if (!url) return env.DB + const authToken = env.LIBSQL_DB_AUTH_TOKEN?.trim() + return libsqlDatabase(createClient({ url, authToken: authToken || undefined })) +} + +/** Envs this module has already produced, so nested handlers (`mono`) don't wrap twice. */ +const resolvedEnvs = new WeakSet() + +/** `env` with `DB` swapped for whichever database {@link openDatabase} picks. */ +export function resolveDatabaseEnv(env: E): E { + if (resolvedEnvs.has(env) || !env.LIBSQL_DB_URL?.trim()) return env + const resolved: E = { ...env, DB: openDatabase(env) } + resolvedEnvs.add(resolved) + return resolved +} + +/** The handler shapes a worker exports: a Hono app, or the `{ fetch, scheduled }` object. */ +export interface DatabaseHandler { + fetch(request: Request, env: E, ctx: ExecutionContext): Response | Promise + scheduled?(controller: ScheduledController, env: E, ctx: ExecutionContext): void | Promise + queue?(batch: MessageBatch, env: E, ctx: ExecutionContext): void | Promise +} + +/** + * What {@link withDatabase} returns: an `ExportedHandler` whose `fetch` is known to exist. + * Declared as a method taking a plain `Request` so a worker that mounts others (`mono`) + * can call it with the request it has, wrapped and unwrapped apps alike. + */ +export interface DatabaseExportedHandler extends ExportedHandler { + fetch(request: Request, env: E, ctx: ExecutionContext): Response | Promise +} + +/** + * Wrap a worker's default export so every handler sees the database + * {@link openDatabase} picks in `env.DB`. This is the ONE place a worker changes to gain + * libSQL support: `export default withDatabase(app)`. + * + * With no `LIBSQL_DB_URL` the env passes through untouched, so a D1 deployment (and every + * test) pays nothing for this. + */ +export function withDatabase( + handler: DatabaseHandler +): DatabaseExportedHandler { + const fetch = handler.fetch.bind(handler) + const wrapped: DatabaseExportedHandler = { + fetch: (request, env, ctx) => fetch(request, resolveDatabaseEnv(env), ctx), + } + if (handler.scheduled) { + const scheduled = handler.scheduled.bind(handler) + wrapped.scheduled = (controller, env, ctx) => + scheduled(controller, resolveDatabaseEnv(env), ctx) + } + if (handler.queue) { + const queue = handler.queue.bind(handler) + wrapped.queue = (batch, env, ctx) => queue(batch, resolveDatabaseEnv(env), ctx) + } + return wrapped +} + +/** A libSQL client behind the D1 API. Exported so a test can hand it a local client. */ +export function libsqlDatabase(client: Client): D1Database { + return new LibsqlDatabase(client) +} + +/** + * D1 accepts null, number, string, boolean and ArrayBuffer; libSQL accepts all of those + * (plus a few more). `undefined` is a bug at the call site on either backend — D1 refuses + * it with a type error — so refuse it here with the same clarity instead of letting the + * client turn it into something. + */ +function toInValue(value: unknown, index: number): InValue { + if (value === undefined) { + throw new TypeError(`bound parameter ${index + 1} is undefined (D1 would reject it too)`) + } + return value as InValue +} + +/** + * The client's rows carry their columns as named properties already, but they are hrana + * `Row` objects with a hidden `length` and numeric indices. Build plain objects by column + * position instead so a row spreads, serialises and compares like the plain object D1 + * hands back — and so a column with no name in the object (a duplicate) is still last-wins + * rather than dropped. + */ +function rowToObject(columns: string[], row: ResultSet['rows'][number]): Record { + const out: Record = {} + for (let i = 0; i < columns.length; i++) out[columns[i]!] = row[i] + return out +} + +function toResult(rs: ResultSet): D1Result { + return { + success: true, + results: rs.rows.map((row) => rowToObject(rs.columns, row) as T), + meta: { + duration: 0, + size_after: 0, + rows_read: 0, + rows_written: 0, + last_row_id: Number(rs.lastInsertRowid ?? 0), + changed_db: rs.rowsAffected > 0, + changes: rs.rowsAffected, + }, + } +} + +class LibsqlPreparedStatement implements D1PreparedStatement { + constructor( + private readonly client: Client, + private readonly sql: string, + private readonly args: InValue[] = [] + ) {} + + /** What the client runs. `batch` collects these from every statement it is handed. */ + toStatement(): InStatement { + return { sql: this.sql, args: this.args } + } + + bind(...values: unknown[]): D1PreparedStatement { + return new LibsqlPreparedStatement(this.client, this.sql, values.map(toInValue)) + } + + private execute(): Promise { + return this.client.execute(this.toStatement()) + } + + first(colName: string): Promise + first>(): Promise + async first(colName?: string): Promise { + const rs = await this.execute() + const row = rs.rows[0] + if (row === undefined) return null + const object = rowToObject(rs.columns, row) + if (colName === undefined) return object as T + if (!rs.columns.includes(colName)) { + throw new Error(`D1_COLUMN_NOTFOUND: column "${colName}" is not in the result`) + } + return object[colName] as T + } + + async run>(): Promise> { + return toResult(await this.execute()) + } + + async all>(): Promise> { + return toResult(await this.execute()) + } + + raw(options: { columnNames: true }): Promise<[string[], ...T[]]> + raw(options?: { columnNames?: false }): Promise + async raw(options?: { columnNames?: boolean }): Promise { + const rs = await this.execute() + const rows = rs.rows.map((row) => rs.columns.map((_, i) => row[i]) as T) + return options?.columnNames ? [rs.columns, ...rows] : rows + } +} + +class LibsqlDatabase implements D1Database { + constructor(private readonly client: Client) {} + + prepare(query: string): D1PreparedStatement { + return new LibsqlPreparedStatement(this.client, query) + } + + /** One write transaction, like D1's: any statement failing rolls back the lot. */ + async batch(statements: D1PreparedStatement[]): Promise>> { + const results = await this.client.batch( + statements.map((statement) => { + if (!(statement instanceof LibsqlPreparedStatement)) { + throw new TypeError('batch: every statement must come from this database') + } + return statement.toStatement() + }), + 'write' + ) + return results.map((rs) => toResult(rs)) + } + + /** A script of `;`-separated statements, as D1's `exec` takes. Nothing here uses it. */ + async exec(query: string): Promise { + const started = Date.now() + await this.client.executeMultiple(query) + const count = query.split(';').filter((statement) => statement.trim() !== '').length + return { count, duration: Date.now() - started } + } + + /** + * D1 sessions pin reads after a write to a replica that has seen it. A libSQL primary + * is always read-your-writes, so a "session" is the database itself with no bookmark. + */ + withSession(): D1DatabaseSession { + return { + prepare: (query) => this.prepare(query), + batch: (statements) => this.batch(statements), + getBookmark: () => null, + } + } + + dump(): Promise { + return Promise.reject(new Error('dump() is not supported on a libSQL database')) + } +} diff --git a/packages/domain/vitest.config.node.ts b/packages/domain/vitest.config.node.ts new file mode 100644 index 0000000..74dfc9b --- /dev/null +++ b/packages/domain/vitest.config.node.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config' + +// Node, not the Workers pool: the libSQL adapter is exercised against an in-memory libSQL +// database through the Node client, which workerd can't host. Everything else in this +// package is covered by the workers' own integration tests, on D1. +export default defineConfig({ + test: { + environment: 'node', + }, +}) diff --git a/packages/hono-helpers/src/types.ts b/packages/hono-helpers/src/types.ts index 88036c1..c65ace0 100644 --- a/packages/hono-helpers/src/types.ts +++ b/packages/hono-helpers/src/types.ts @@ -20,6 +20,14 @@ export type SharedHonoEnv = { * Useful for logs, Sentry, etc. */ SENTRY_RELEASE: string + /** + * A libSQL server (Turso) to use INSTEAD of the D1 binding, e.g. `libsql://-.turso.io`. + * Set by `just deploy` from `RECFLARE_LIBSQL_DB_URL`; unset means D1. Read only by + * `withDatabase` in @repo/domain, which swaps `DB` for the matching adapter. + */ + LIBSQL_DB_URL?: string + /** That server's auth token — a worker SECRET `just deploy` sets, never a var. */ + LIBSQL_DB_AUTH_TOKEN?: string } /** Global Hono variables */ export type SharedHonoVariables = { diff --git a/packages/tools/bin/run-wrangler-deploy b/packages/tools/bin/run-wrangler-deploy index 5d4dd5f..4401e3f 100755 --- a/packages/tools/bin/run-wrangler-deploy +++ b/packages/tools/bin/run-wrangler-deploy @@ -181,6 +181,20 @@ fi # both a deploy and a local dev server. EXTRA_VARS=$(recflare_vars) +# Using a libSQL server (Turso) instead of D1: RECFLARE_LIBSQL_DB_URL reaches the worker as the +# LIBSQL_DB_URL var and switches `withDatabase` (@repo/domain) over; the D1 binding stays in +# place, unused, so pointing back at D1 is unsetting the URL and redeploying. The token is NOT +# a var — it is put as an encrypted worker secret after the deploy (secrets survive a deploy, +# vars are replaced wholesale). Both come from .env; see .env.example. +LIBSQL_VARS="" +if [ -n "${RECFLARE_LIBSQL_DB_URL:-}" ]; then + if [ -z "${RECFLARE_LIBSQL_DB_AUTH_TOKEN:-}" ]; then + echo "error: RECFLARE_LIBSQL_DB_URL is set but RECFLARE_LIBSQL_DB_AUTH_TOKEN is not — add the database's auth token to .env (see .env.example)" >&2 + exit 1 + fi + LIBSQL_VARS="--var LIBSQL_DB_URL:$RECFLARE_LIBSQL_DB_URL" +fi + # Vite-built configs set no_bundle (vite already bundled and minified), which is # incompatible with --minify. Only pass --minify when wrangler does the bundling. # Keyed on IS_VITE, not on $CONFIG: the splicing above may have swapped $CONFIG for @@ -199,6 +213,14 @@ wrangler deploy \ --var DOMAIN:"$DOMAIN" \ --var SUBDOMAINS:"$SUBDOMAINS_JSON" \ $EXTRA_VARS \ + $LIBSQL_VARS \ --domain "$HOST" \ $MINIFY \ "$@" + +# The token, once the worker exists (a first deploy has nothing to attach a secret to). On the +# very first switch to libSQL this leaves a few seconds in which the new code has a URL and no +# token; every later deploy finds the secret already there. +if [ -n "${RECFLARE_LIBSQL_DB_URL:-}" ]; then + printf '%s' "$RECFLARE_LIBSQL_DB_AUTH_TOKEN" | wrangler secret put LIBSQL_DB_AUTH_TOKEN --config "$CONFIG" +fi diff --git a/packages/tools/bin/run-wrangler-migrate b/packages/tools/bin/run-wrangler-migrate index 25c754d..d82eae3 100755 --- a/packages/tools/bin/run-wrangler-migrate +++ b/packages/tools/bin/run-wrangler-migrate @@ -46,6 +46,13 @@ fi # RECFLARE_D1 comes from the gitignored root .env, or from CI secrets, which win over it. recflare_load_env +# A libSQL server (Turso) configured in .env takes the remote migrations instead of D1: the +# same files, the same per-worker migrations table, applied by `runx libsql migrate` from +# packages/tools. The D1 database is not touched. `--local` above never comes here. +if [ -n "${RECFLARE_LIBSQL_DB_URL:-}" ]; then + exec runx libsql migrate +fi + DB_ID=${RECFLARE_D1:-} if [ -z "$DB_ID" ]; then echo "error: RECFLARE_D1 is not set — add the recflare D1 id to .env (see .env.example)" >&2 diff --git a/packages/tools/package.json b/packages/tools/package.json index c2cc1fd..5a4010b 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -19,6 +19,7 @@ "dependencies": { "@commander-js/extra-typings": "15.0.0", "@jahands/cli-tools": "0.11.5", + "@libsql/client": "0.18.0", "@types/fs-extra": "11.0.4", "@types/node": "26.0.1", "@typescript/native-preview": "7.0.0-dev.20260629.1", diff --git a/packages/tools/src/bin/runx.cmd.ts b/packages/tools/src/bin/runx.cmd.ts index 437f2a0..11b0f4d 100644 --- a/packages/tools/src/bin/runx.cmd.ts +++ b/packages/tools/src/bin/runx.cmd.ts @@ -10,6 +10,7 @@ import { checkCmd } from '../cmd/check.cmd' import { ciCmd } from '../cmd/ci.cmd' import { devCmd } from '../cmd/dev.cmd' import { fixCmd } from '../cmd/fix.cmd' +import { libsqlCmd } from '../cmd/libsql.cmd' import { shfmtCmd } from '../cmd/shfmt.cmd' import { storefrontCmd } from '../cmd/storefront.cmd' import { updateCmd } from '../cmd/update.cmd' @@ -25,6 +26,7 @@ program .addCommand(catalogCmd) .addCommand(storefrontCmd) .addCommand(fixCmd) + .addCommand(libsqlCmd) .addCommand(buildCmd) .addCommand(checkCmd) .addCommand(devCmd) diff --git a/packages/tools/src/cmd/libsql.cmd.ts b/packages/tools/src/cmd/libsql.cmd.ts new file mode 100644 index 0000000..e2f1d27 --- /dev/null +++ b/packages/tools/src/cmd/libsql.cmd.ts @@ -0,0 +1,74 @@ +import { Command } from '@commander-js/extra-typings' + +import { libsqlClient, readLibsqlTarget } from '../d1' + +/** + * `runx libsql migrate` — apply a worker's D1 migrations to the libSQL server (Turso) + * configured in .env, which is what `just migrate` runs when RECFLARE_LIBSQL_DB_URL is set + * (see run-wrangler-migrate). `wrangler d1 migrations apply` only knows D1, so this does the + * same job by hand, the same way: + * + * - the files are the worker's `migrations_dir` (from its wrangler.jsonc), `.sql`, applied + * in name order; + * - what has been applied is recorded in the worker's `migrations_table`, created with + * wrangler's exact schema. A D1 database exported and loaded into libSQL therefore + * brings its history along, and this picks up where D1 left off — and a libSQL database + * loaded back into D1 hands wrangler a table it recognises; + * - each file runs in one transaction with its history row, so a failed migration leaves + * nothing half-applied and is simply re-run once fixed. + * + * Run from the worker's directory (turbo does; `--dir` says otherwise). + */ +export const libsqlCmd = new Command('libsql').description( + 'Operate on the libSQL (Turso) database configured in .env, in place of D1' +) + +libsqlCmd + .command('migrate') + .description("Apply this worker's pending migrations to the configured libSQL server") + .option('--dir ', 'the worker directory (defaults to the current one)') + .action(async (opts) => { + const target = await readLibsqlTarget() + if (!target) { + throw new Error('RECFLARE_LIBSQL_DB_URL is not set — nothing to migrate (see .env.example)') + } + const dir = path.resolve(opts.dir ?? process.cwd()) + const config = await fs.readFile(path.join(dir, 'wrangler.jsonc'), 'utf8') + const migrationsDir = config.match(/"migrations_dir"\s*:\s*"([^"]+)"/)?.[1] ?? 'migrations' + const table = config.match(/"migrations_table"\s*:\s*"([^"]+)"/)?.[1] ?? 'd1_migrations' + if (!/^\w+$/.test(table)) throw new Error(`unusable migrations_table name: ${table}`) + + const files = (await fs.readdir(path.join(dir, migrationsDir))) + .filter((name) => name.endsWith('.sql')) + .sort() + + const client = libsqlClient(target) + await client.execute( + `CREATE TABLE IF NOT EXISTS "${table}" (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE, applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)` + ) + const applied = new Set( + (await client.execute(`SELECT name FROM "${table}"`)).rows + .map((row) => row.name) + .filter((name): name is string => typeof name === 'string') + ) + const pending = files.filter((name) => !applied.has(name)) + const worker = path.basename(dir) + if (pending.length === 0) { + console.log(chalk.dim(`${worker}: no pending migrations (${table})`)) + return + } + + for (const name of pending) { + const sql = await fs.readFile(path.join(dir, migrationsDir, name), 'utf8') + const tx = await client.transaction('write') + try { + await tx.executeMultiple(sql) + await tx.execute({ sql: `INSERT INTO "${table}" (name) VALUES (?)`, args: [name] }) + await tx.commit() + } catch (err) { + await tx.rollback().catch(() => {}) + throw new Error(`${worker}: migration ${name} failed: ${String(err)}`) + } + console.log(`${worker}: applied ${chalk.green(name)}`) + } + }) diff --git a/packages/tools/src/d1.ts b/packages/tools/src/d1.ts index 3a4febf..d8c71e4 100644 --- a/packages/tools/src/d1.ts +++ b/packages/tools/src/d1.ts @@ -1,7 +1,11 @@ import 'zx/globals' +import { createClient } from '@libsql/client/web' + import { getRepoRoot } from './path' +import type { Client, ResultSet } from '@libsql/client/web' + /** * Shared plumbing for talking to the one `recflare` D1 database from a CLI. * @@ -13,6 +17,11 @@ import { getRepoRoot } from './path' * needs the real D1 id, because the committed `wrangler.jsonc` files all carry the literal * placeholder `"local"` as their `database_id`. Splicing the real id into a gitignored * generated config is the same thing `run-wrangler-migrate` does at deploy time. + * + * With a libSQL server (Turso) configured — RECFLARE_LIBSQL_DB_URL in .env, see .env.example — + * `--remote` means THAT database, reached directly through `@libsql/client` rather than through + * wrangler, so an operator's `runx admin --remote` acts on the database the deployed workers + * are actually using. `--local` is the local D1 either way. */ /** The one shared database every D1-backed worker binds. */ @@ -112,6 +121,10 @@ export async function execSql>( remote: boolean, worker = 'auth' ): Promise> { + if (remote) { + const target = await readLibsqlTarget() + if (target) return libsqlResult(await libsqlClient(target).execute(sql)) + } const stdout = await runD1(worker, ['--command', sql, '--json'], remote) // wrangler --json prints a one-element array of results to stdout. const start = stdout.indexOf('[') @@ -130,5 +143,59 @@ export async function execSql>( * which is megabytes of nothing. The human-readable output is returned for the caller. */ export async function execSqlFile(file: string, remote: boolean, worker = 'auth'): Promise { + if (remote) { + const target = await readLibsqlTarget() + if (target) { + // One script in one request. Fine for the loads this is used for; a file too large for + // the server's request limit would have to be split by statement first. + await libsqlClient(target).executeMultiple(await fs.readFile(file, 'utf8')) + return `applied ${path.basename(file)} to ${new URL(target.url).host}` + } + } return await runD1(worker, ['--file', file], remote) } + +/** Where `--remote` goes when the deployment runs on libSQL instead of D1. */ +export interface LibsqlTarget { + url: string + authToken?: string +} + +/** + * The libSQL target from RECFLARE_LIBSQL_DB_URL / RECFLARE_LIBSQL_DB_AUTH_TOKEN, or undefined + * when the deployment is on D1 (the URL unset). Same precedence as every other setting: + * process environment first, then the root .env. + */ +export async function readLibsqlTarget(): Promise { + const url = await readRootEnv('RECFLARE_LIBSQL_DB_URL') + if (!url) return undefined + return { url, authToken: await readRootEnv('RECFLARE_LIBSQL_DB_AUTH_TOKEN') } +} + +let libsql: { url: string; client: Client } | undefined + +/** + * A client for the target, made once per process. The web build of the client is used on + * purpose: it speaks Hrana over HTTP with nothing native to install, and a CLI only ever + * talks to a remote server. + */ +export function libsqlClient(target: LibsqlTarget): Client { + if (libsql?.url !== target.url) { + console.error(chalk.magenta(`→ libSQL ${new URL(target.url).host}`)) + libsql = { + url: target.url, + client: createClient({ url: target.url, authToken: target.authToken }), + } + } + return libsql.client +} + +/** The client's rows as the plain objects wrangler's `--json` output gives. */ +function libsqlResult(rs: ResultSet): D1ExecResult { + const results = rs.rows.map((row) => { + const out: Record = {} + rs.columns.forEach((column, i) => (out[column] = row[i])) + return out as Row + }) + return { results, success: true, meta: { changes: rs.rowsAffected } } +} diff --git a/packages/tools/src/sh/env.sh b/packages/tools/src/sh/env.sh index bfa5a62..b6f3995 100644 --- a/packages/tools/src/sh/env.sh +++ b/packages/tools/src/sh/env.sh @@ -12,8 +12,11 @@ # and `just dev`, so a value is never configured twice. # Names the deploy scripts consume themselves — the domain and the ids of the operator's -# Cloudflare resources. Everything else in .env is worker config; see recflare_vars. -RECFLARE_RESERVED="DOMAIN SUBDOMAINS D1 KV SECRETS_STORE ENV_LOADED" +# Cloudflare resources. The two LIBSQL_DB_* entries are reserved too: they choose the database (see .env.example) and the deploy script hands them +# over itself — the URL as a var, the token as an encrypted worker SECRET — so neither may +# ride along as a plain `--var`, nor reach a local dev server, which stays on the local D1. +# Everything else in .env is worker config; see recflare_vars. +RECFLARE_RESERVED="DOMAIN SUBDOMAINS D1 KV SECRETS_STORE LIBSQL_DB_URL LIBSQL_DB_AUTH_TOKEN ENV_LOADED" # Load the root .env, letting anything already in the environment win. The file is a local # convenience; CI exports the same names as secrets and must not be clobbered by a stray diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fcab0c9..957244c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -347,6 +347,9 @@ importers: '@2toad/profanity': specifier: 3.3.0 version: 3.3.0 + '@repo/domain': + specifier: workspace:* + version: link:../../packages/domain '@repo/hono-helpers': specifier: workspace:* version: link:../../packages/hono-helpers @@ -629,6 +632,9 @@ importers: '@cf-wasm/photon': specifier: ^0.3.6 version: 0.3.6 + '@repo/domain': + specifier: workspace:* + version: link:../../packages/domain '@repo/hono-helpers': specifier: workspace:* version: link:../../packages/hono-helpers @@ -1304,6 +1310,9 @@ importers: packages/domain: dependencies: + '@libsql/client': + specifier: 0.18.0 + version: 0.18.0 isemail: specifier: ^3.2.0 version: 3.2.0 @@ -1317,6 +1326,9 @@ importers: '@repo/typescript-config': specifier: workspace:* version: link:../typescript-config + vitest: + specifier: 4.1.9 + version: 4.1.9(@opentelemetry/api@1.9.0)(@types/node@26.0.1)(vite@6.4.3(@types/node@26.0.1)(jiti@2.6.1)(lightningcss@1.29.2)(tsx@4.22.4)(yaml@2.9.0)) packages/hono-helpers: dependencies: @@ -1395,6 +1407,9 @@ importers: '@jahands/cli-tools': specifier: 0.11.5 version: 0.11.5(@commander-js/extra-typings@15.0.0(commander@15.0.0))(commander@15.0.0)(typescript@6.0.3)(zod@4.4.3)(zx@8.8.5) + '@libsql/client': + specifier: 0.18.0 + version: 0.18.0 '@types/fs-extra': specifier: 11.0.4 version: 11.0.4 @@ -2663,9 +2678,69 @@ packages: '@lezer/yaml@1.0.4': resolution: {integrity: sha512-2lrrHqxalACEbxIbsjhqGpSW8kWpUKuY6RHgnSAFZa6qK62wvnPxA8hGOwOoDbwHcOFs5M4o27mjGu+P7TvBmw==} + '@libsql/client@0.18.0': + resolution: {integrity: sha512-zMCCo58vv7w27j7+cnt/GW+X6/Rih6cHZ86PKi0AxCMOqMfrVgG9OZ5lW2bDMJhSeOjsv2szH2nJF3A9l36UmA==} + + '@libsql/core@0.18.0': + resolution: {integrity: sha512-N8RR2CIpcgtbKZnXs2KVpw2lJQfinUj3I56e7qRDRpK18Hty0Rr8nr00VmFb4R/DsqOanPB/TaP3+QAQLZeXMg==} + + '@libsql/darwin-arm64@0.5.29': + resolution: {integrity: sha512-K+2RIB1OGFPYQbfay48GakLhqf3ArcbHqPFu7EZiaUcRgFcdw8RoltsMyvbj5ix2fY0HV3Q3Ioa/ByvQdaSM0A==} + cpu: [arm64] + os: [darwin] + + '@libsql/darwin-x64@0.5.29': + resolution: {integrity: sha512-OtT+KFHsKFy1R5FVadr8FJ2Bb1mghtXTyJkxv0trocq7NuHntSki1eUbxpO5ezJesDvBlqFjnWaYYY516QNLhQ==} + cpu: [x64] + os: [darwin] + + '@libsql/hrana-client@0.10.0': + resolution: {integrity: sha512-OoA4EMqRAC7kn7V2P6EQqRcpZf2W+AjsNIyCizBg339Tq/aMC7sRnzs3SklderhmQWAqEzvv8A2vhxVmWpkVvw==} + + '@libsql/isomorphic-ws@0.1.5': + resolution: {integrity: sha512-DtLWIH29onUYR00i0GlQ3UdcTRC6EP4u9w/h9LxpUZJWRMARk6dQwZ6Jkd+QdwVpuAOrdxt18v0K2uIYR3fwFg==} + + '@libsql/linux-arm-gnueabihf@0.5.29': + resolution: {integrity: sha512-CD4n4zj7SJTHso4nf5cuMoWoMSS7asn5hHygsDuhRl8jjjCTT3yE+xdUvI4J7zsyb53VO5ISh4cwwOtf6k2UhQ==} + cpu: [arm] + os: [linux] + + '@libsql/linux-arm-musleabihf@0.5.29': + resolution: {integrity: sha512-2Z9qBVpEJV7OeflzIR3+l5yAd4uTOLxklScYTwpZnkm2vDSGlC1PRlueLaufc4EFITkLKXK2MWBpexuNJfMVcg==} + cpu: [arm] + os: [linux] + + '@libsql/linux-arm64-gnu@0.5.29': + resolution: {integrity: sha512-gURBqaiXIGGwFNEaUj8Ldk7Hps4STtG+31aEidCk5evMMdtsdfL3HPCpvys+ZF/tkOs2MWlRWoSq7SOuCE9k3w==} + cpu: [arm64] + os: [linux] + + '@libsql/linux-arm64-musl@0.5.29': + resolution: {integrity: sha512-fwgYZ0H8mUkyVqXZHF3mT/92iIh1N94Owi/f66cPVNsk9BdGKq5gVpoKO+7UxaNzuEH1roJp2QEwsCZMvBLpqg==} + cpu: [arm64] + os: [linux] + + '@libsql/linux-x64-gnu@0.5.29': + resolution: {integrity: sha512-y14V0vY0nmMC6G0pHeJcEarcnGU2H6cm21ZceRkacWHvQAEhAG0latQkCtoS2njFOXiYIg+JYPfAoWKbi82rkg==} + cpu: [x64] + os: [linux] + + '@libsql/linux-x64-musl@0.5.29': + resolution: {integrity: sha512-gquqwA/39tH4pFl+J9n3SOMSymjX+6kZ3kWgY3b94nXFTwac9bnFNMffIomgvlFaC4ArVqMnOZD3nuJ3H3VO1w==} + cpu: [x64] + os: [linux] + + '@libsql/win32-x64-msvc@0.5.29': + resolution: {integrity: sha512-4/0CvEdhi6+KjMxMaVbFM2n2Z44escBRoEYpR+gZg64DdetzGnYm8mcNLcoySaDJZNaBd6wz5DNdgRmcI4hXcg==} + cpu: [x64] + os: [win32] + '@marijn/find-cluster-break@1.0.4': resolution: {integrity: sha512-Wy0V7+SGUjnF9/TkiM1hKVDPj7jKXduPNboMVtHTA8dySMURWqfg/JZ9E2Sq8JgSJmkl7k7Qe9FLeMSrSraWmQ==} + '@neon-rs/load@0.0.4': + resolution: {integrity: sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==} + '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} @@ -3249,6 +3324,9 @@ packages: '@types/web-bluetooth@0.0.21': resolution: {integrity: sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA==} + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260629.1': resolution: {integrity: sha512-wXRExZJweYoTzE4atRR7T5HwKJYkl6/KHxON0eF0iy2fvgLXDlyq4AQqhmV8mMx10PQKc/4sNbfhD4kjWWvm8A==} engines: {node: '>=16.20.0'} @@ -3578,6 +3656,10 @@ packages: resolution: {integrity: sha512-y+8xyqdGLL+6sh0tVeHcfP/QDd8gUgbasolJJpY7NgeQGSZ739bDtSiaiDgtoicy+mtYB81dKLxO9xRhCyIB3A==} engines: {node: '>=12.20'} + detect-libc@2.0.2: + resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==} + engines: {node: '>=8'} + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -3850,6 +3932,11 @@ packages: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} + libsql@0.5.29: + resolution: {integrity: sha512-8lMP8iMgiBzzoNbAPQ59qdVcj6UaE/Vnm+fiwX4doX4Narook0a4GPKWBEv+CR8a1OwbfkgL18uBfBjWdF0Fzg==} + cpu: [x64, arm64, wasm32, arm] + os: [darwin, linux, win32] + lightningcss-darwin-arm64@1.29.2: resolution: {integrity: sha512-cK/eMabSViKn/PG8U/a7aCorpeKLMlK0bQeNHmdb7qUnBkNPnL+oV5DjJUo0kqWsJUapZsM4jCfYItbqBDvlcA==} engines: {node: '>= 12.0.0'} @@ -4189,6 +4276,9 @@ packages: resolution: {integrity: sha512-HzMy3Geq23nVALD/M2LliU+F+M+gVNsvkQWWqeBZ8HDiCgzo6YPJ/Omrmtq24EFrIsk0a3EkQGEd7bDOo+IhGA==} engines: {node: '>=18'} + promise-limit@2.7.0: + resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==} + property-information@7.2.0: resolution: {integrity: sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==} @@ -5741,8 +5831,68 @@ snapshots: '@lezer/highlight': 1.2.3 '@lezer/lr': 1.4.10 + '@libsql/client@0.18.0': + dependencies: + '@libsql/core': 0.18.0 + '@libsql/hrana-client': 0.10.0 + js-base64: 3.9.3 + libsql: 0.5.29 + promise-limit: 2.7.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@libsql/core@0.18.0': + dependencies: + js-base64: 3.9.3 + + '@libsql/darwin-arm64@0.5.29': + optional: true + + '@libsql/darwin-x64@0.5.29': + optional: true + + '@libsql/hrana-client@0.10.0': + dependencies: + '@libsql/isomorphic-ws': 0.1.5 + js-base64: 3.9.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@libsql/isomorphic-ws@0.1.5': + dependencies: + '@types/ws': 8.18.1 + ws: 8.21.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@libsql/linux-arm-gnueabihf@0.5.29': + optional: true + + '@libsql/linux-arm-musleabihf@0.5.29': + optional: true + + '@libsql/linux-arm64-gnu@0.5.29': + optional: true + + '@libsql/linux-arm64-musl@0.5.29': + optional: true + + '@libsql/linux-x64-gnu@0.5.29': + optional: true + + '@libsql/linux-x64-musl@0.5.29': + optional: true + + '@libsql/win32-x64-msvc@0.5.29': + optional: true + '@marijn/find-cluster-break@1.0.4': {} + '@neon-rs/load@0.0.4': {} + '@opentelemetry/api@1.9.0': {} '@oxlint-tsgolint/darwin-arm64@0.23.0': @@ -6391,6 +6541,10 @@ snapshots: '@types/web-bluetooth@0.0.21': {} + '@types/ws@8.18.1': + dependencies: + '@types/node': 26.0.1 + '@typescript/native-preview-darwin-arm64@7.0.0-dev.20260629.1': optional: true @@ -6686,6 +6840,8 @@ snapshots: detect-indent@7.0.2: {} + detect-libc@2.0.2: {} + detect-libc@2.1.2: {} detect-newline@4.0.1: {} @@ -7009,6 +7165,21 @@ snapshots: kleur@4.1.5: {} + libsql@0.5.29: + dependencies: + '@neon-rs/load': 0.0.4 + detect-libc: 2.0.2 + optionalDependencies: + '@libsql/darwin-arm64': 0.5.29 + '@libsql/darwin-x64': 0.5.29 + '@libsql/linux-arm-gnueabihf': 0.5.29 + '@libsql/linux-arm-musleabihf': 0.5.29 + '@libsql/linux-arm64-gnu': 0.5.29 + '@libsql/linux-arm64-musl': 0.5.29 + '@libsql/linux-x64-gnu': 0.5.29 + '@libsql/linux-x64-musl': 0.5.29 + '@libsql/win32-x64-msvc': 0.5.29 + lightningcss-darwin-arm64@1.29.2: optional: true @@ -7525,6 +7696,8 @@ snapshots: dependencies: parse-ms: 4.0.0 + promise-limit@2.7.0: {} + property-information@7.2.0: {} punycode@2.3.1: {}