[rooms] improve performance of rooms query

apps/rooms/migrations/0014_room_listable.sql

@@ -0,0 +1,34 @@
+-- The room flags every public feed filters on, as indexed columns.
+--
+-- Generated from packages/domain/src/rooms-db.ts (ROOM_SCHEMA_DDL) — keep in sync.
+--
+-- `is_dorm` was already extracted back in 0001, but `Accessibility` and `ExcludeFromLists`
+-- were not, so "the rooms a feed may show" could only be asked in JS. Every feed — hot, recommended,
+-- featured, new, recently-updated, similar, search, autocomplete — therefore SELECTed the
+-- whole `room` table and threw most of it away after parsing it. On this database that is
+-- 1173 rows and ~1.7MB of blob read to rank the 109 rooms that are actually listable: the
+-- rest are DORMS (one per account, private by construction) and other private rooms. It is
+-- the read D1 reports as slow, and it gets worse with every account that signs up.
+--
+-- VIRTUAL like the other generated columns, so the blob stays the only copy of the value
+-- and no backfill is needed — they are computed on read from the JSON already stored.
+ALTER TABLE room ADD COLUMN accessibility INTEGER GENERATED ALWAYS AS (json_extract(data, '$.Accessibility')) VIRTUAL;
+ALTER TABLE room ADD COLUMN exclude_from_lists INTEGER GENERATED ALWAYS AS (json_extract(data, '$.ExcludeFromLists')) VIRTUAL;
+
+-- A PARTIAL index: it holds only the public, non-dorm rooms, which is the small minority
+-- the feeds serve from. Scanning it visits those rows alone, so the feeds stop reading the
+-- dorms at all rather than reading and discarding them. EXPLAIN QUERY PLAN over the feed
+-- query goes from
+--
+--   SCAN room
+--
+-- to
+--
+--   SCAN room USING INDEX idx_room_public
+--
+-- `ExcludeFromLists` is deliberately NOT in the WHERE: search reads the public rooms
+-- WITHOUT that term (a room can opt out of the browse feeds and still be findable by
+-- name), so indexing the wider set lets both reads use this one index — the extra term is
+-- then checked against the handful of rows it already fetched.
+CREATE INDEX IF NOT EXISTS idx_room_public ON room (room_id)
+  WHERE is_dorm IS NOT 1 AND accessibility = 1;

apps/rooms/src/test/integration/api.test.ts

@@ -1053,6 +1053,54 @@ describe('rooms endpoints', () => {
 		expect(body.length).toBeLessThanOrEqual(3)
 	})
 
+	// The feeds narrow to the listable rooms in SQL (LISTABLE_WHERE, off idx_room_public)
+	// rather than by parsing every room in the database. That predicate has to answer
+	// exactly what the in-memory `isListable` answered, and the interesting case is the
+	// MISSING key: `ExcludeFromLists !== true` accepts a blob that never carried the field,
+	// so the SQL has to accept the NULL its generated column extracts.
+	it('the feeds’ SQL filter treats a missing ExcludeFromLists as not-excluded', async () => {
+		const seed = (data: Record<string, unknown>) =>
+			env.DB.prepare('INSERT INTO room (data) VALUES (?1)').bind(JSON.stringify(data)).run()
+
+		// No `ExcludeFromLists` key at all — the shape of every room seeded before the flag
+		// existed.
+		await seed({
+			RoomId: 30801,
+			Name: 'NoExcludeKey',
+			CreatorAccountId: 830,
+			Accessibility: 1,
+			IsDorm: false,
+			SubRooms: [],
+		})
+		// Same room, opted out.
+		await seed({
+			RoomId: 30802,
+			Name: 'OptedOut',
+			CreatorAccountId: 830,
+			Accessibility: 1,
+			IsDorm: false,
+			ExcludeFromLists: true,
+			SubRooms: [],
+		})
+
+		const namesIn = async (path: string) => {
+			const body = (await (await SELF.fetch(`${ORIGIN}${path}`)).json()) as
+				{ Results: Array<{ Name: string }> } | Array<{ Name: string }>
+			return (Array.isArray(body) ? body : body.Results).map((r) => r.Name)
+		}
+
+		for (const feed of ['/rooms/hot?take=200', '/rooms/recommendations?take=200']) {
+			expect(await namesIn(feed)).toContain('NoExcludeKey')
+			expect(await namesIn(feed)).not.toContain('OptedOut')
+		}
+
+		// SEARCH is the wider filter (PUBLIC_WHERE): a room can opt out of the browse feeds
+		// and still be findable by name, so this must not have narrowed with the feeds.
+		expect(await namesIn('/rooms/search?query=optedout')).toContain('OptedOut')
+
+		await env.DB.prepare('DELETE FROM room WHERE room_id IN (30801, 30802)').run()
+	})
+
 	// Served only to the client builds that render it — the 2023 client's other room
 	// listings start failing with NREs when it gets this payload, which is why the route
 	// was parked entirely for a while (see FEATURED_ROOMS_VERSIONS in rooms.app.ts).