[studio] Complete editor uploads and queued cloud builds (#135)

* [studio] Complete editor uploads and queued cloud builds

* [studio] Protect unpublished source and remove unused notification RPC

apps/storage/src/storage.app.ts

@@ -51,6 +51,9 @@ const UPLOAD_SUBFOLDER: Record<number, string> = {
 	4: 'video',
 	5: 'invention',
 	6: 'room',
+	8: 'room',
+	9: 'roomunityasset',
+	10: 'unityasset',
 }
 
 /** Resolve the storage subfolder for a posted FileType, or `undefined` when unknown. */
@@ -100,6 +103,121 @@ function textField(body: Record<string, unknown>, ...names: string[]): string |
 	return undefined
 }
 
+/** Workers requires quoted disposition parameters; Mono emits valid bare tokens. */
+function normalizeMultipartHeaders(contentType: string): TransformStream<Uint8Array, Uint8Array> {
+	const boundary = /boundary=(?:"([^"]+)"|([^;\s]+))/i.exec(contentType)
+	const marker = new TextEncoder().encode(`\r\n--${boundary?.[1] ?? boundary?.[2]}`)
+	let pending = new Uint8Array(0)
+	let headers = true
+	return new TransformStream({
+		transform(chunk, controller) {
+			const bytes = new Uint8Array(pending.length + chunk.length)
+			bytes.set(pending)
+			bytes.set(chunk, pending.length)
+			pending = bytes
+			while (pending.length) {
+				if (headers) {
+					let end = -1
+					for (let i = 0; i + 3 < pending.length; i++)
+						if (
+							pending[i] === 13 &&
+							pending[i + 1] === 10 &&
+							pending[i + 2] === 13 &&
+							pending[i + 3] === 10
+						) {
+							end = i + 4
+							break
+						}
+					if (end < 0) {
+						if (pending.length > 64 * 1024) throw new Error('Multipart headers exceed limit')
+						return
+					}
+					const text = new TextDecoder()
+						.decode(pending.subarray(0, end))
+						.replace(/^Content-Disposition:[^\r\n]*/gim, (line) =>
+							line.replace(
+								/(;\s*(?:name|filename)=)([!#$%&'*+\-.^_`|~0-9a-z]+)(?=;|\r|$)/gi,
+								'$1"$2"'
+							)
+						)
+					controller.enqueue(new TextEncoder().encode(text))
+					pending = pending.subarray(end)
+					headers = false
+				} else {
+					let start = -1
+					for (let i = 0; i + marker.length + 1 < pending.length; i++) {
+						if (
+							pending[i] === 13 &&
+							marker.every((byte, j) => pending[i + j] === byte) &&
+							pending[i + marker.length] === 13 &&
+							pending[i + marker.length + 1] === 10
+						) {
+							start = i
+							break
+						}
+					}
+					if (start < 0) {
+						const end = Math.max(0, pending.length - marker.length - 2)
+						if (end) controller.enqueue(pending.subarray(0, end))
+						pending = pending.subarray(end)
+						return
+					}
+					const end = start + marker.length + 2
+					controller.enqueue(pending.subarray(0, end))
+					pending = pending.subarray(end)
+					headers = true
+				}
+			}
+		},
+		flush(controller) {
+			if (pending.length) controller.enqueue(pending)
+		},
+	})
+}
+
+/** Studio compresses the entire multipart body, not the individual file parts. */
+async function readUploadForm(
+	request: Request,
+	limit: number
+): Promise<{ form: FormData } | { error: string; status: 400 | 413 | 415 }> {
+	const encoding = request.headers.get('content-encoding')?.trim().toLowerCase()
+	if (encoding && encoding !== 'identity' && encoding !== 'gzip')
+		return { error: 'Unsupported upload encoding.', status: 415 }
+	let tooLarge = false
+	try {
+		let stream = request.body
+		if (!stream) return { form: new FormData() }
+		if (encoding === 'gzip') stream = stream.pipeThrough(new DecompressionStream('gzip'))
+		let decodedBytes = 0
+		stream = stream.pipeThrough(
+			new TransformStream<Uint8Array, Uint8Array>({
+				transform(chunk, controller) {
+					decodedBytes += chunk.byteLength
+					// Account for the multipart envelope while bounding expanded input before parsing.
+					if (decodedBytes > limit + 64 * 1024) {
+						tooLarge = true
+						throw new Error('Decoded upload exceeds limit')
+					}
+					controller.enqueue(chunk)
+				},
+			})
+		)
+		const headers = new Headers(request.headers)
+		headers.delete('content-encoding')
+		headers.delete('content-length')
+		if (/^multipart\/form-data\s*;/i.test(headers.get('content-type') ?? ''))
+			stream = stream.pipeThrough(normalizeMultipartHeaders(headers.get('content-type')!))
+		return {
+			form: await new Request(request.url, { method: 'POST', headers, body: stream }).formData(),
+		}
+	} catch {
+		return {
+			error: tooLarge ? 'Files exceed the upload limit.' : 'Invalid upload form.',
+			status: tooLarge ? 413 : 400,
+		}
+	}
+}
+
 const app = new Hono<App>()
 	.use(
 		'*',
@@ -140,6 +258,46 @@ const app = new Hono<App>()
 	// (the `<upload-date>/<random-name>` part the `cdn` worker serves back) the client
 	// references it by. Also accepts a name-only post (no binary) that just echoes
 	// back an explicit `name`/`filename`/`imagename`. Mirrors the reference `Upload`.
+	.post(
+		'/upload/unityassets',
+		describeRoute({
+			tags: ['Upload'],
+			summary: 'Upload Studio source files',
+			security: AUTHED,
+			description:
+				'Repeated multipart file parts, returned in request order with filename and SHA-256.',
+		}),
+		async (c) => {
+			const id = await validateAndGetAccountId(c.req.raw, await c.env.JWT_SECRET.get())
+			if (id === null) return c.body(null, 401)
+			const limit = maxUploadBytes(c.env.MAX_UPLOAD_BYTES)
+			const parsed = await readUploadForm(c.req.raw, limit)
+			if ('error' in parsed) return c.json({ error: parsed.error }, parsed.status)
+			const form = parsed.form
+			const files = Array.from(form.values()).filter(
+				(value): value is File => value instanceof File
+			)
+			if (files.length === 0) return c.json({ error: 'No source files supplied.' }, 400)
+			if (files.reduce((size, file) => size + file.size, 0) > limit)
+				return c.json({ error: 'Source files exceed the upload limit.' }, 413)
+			const uploaded = []
+			for (const file of files) {
+				const bytes = await file.arrayBuffer()
+				const hash = await crypto.subtle.digest('SHA-256', bytes)
+				// SDK XYGEAHCXHVR: unsigned little-endian SHA-256, base36 padded to
+				// 50 digits, then the first 40 characters. Its source manifest already
+				// contains this name before upload; random names cannot round-trip.
+				const name = new Uint8Array(hash)
+					.reduceRight((n, byte) => n * 256n + BigInt(byte), 0n)
+					.toString(36)
+					.padStart(50, '0')
+					.slice(0, 40)
+				await c.env.CDN_ASSETS.put(`unityasset/${name}`, bytes, { sha256: hash })
+				uploaded.push({ Filename: name, Hash: btoa(String.fromCharCode(...new Uint8Array(hash))) })
+			}
+			return c.json(uploaded)
+		}
+	)
 	.post(
 		'/upload',
 		describeRoute({
@@ -173,16 +331,18 @@ const app = new Hono<App>()
 			const id = await validateAndGetAccountId(c.req.raw, secret)
 			if (id === null) return c.body(null, 401)
 
-			const body = await c.req.parseBody().catch(() => ({}) as Record<string, unknown>)
+			const roles = await validateAndGetRoles(c.req.raw, secret)
+			const isStaff = roles?.some((role) => STAFF_ROLES.has(role)) ?? false
+			const limit = maxUploadBytes(c.env.MAX_UPLOAD_BYTES)
+			const parsed = await readUploadForm(c.req.raw, isStaff ? Number.MAX_SAFE_INTEGER : limit)
+			if ('error' in parsed) return c.json({ error: parsed.error }, parsed.status)
+			const body = Object.fromEntries(parsed.form.entries())
 
 			// The binary part is identified by being a file (filename/content-type),
 			// not by its field name — matching the reference's part detection.
 			const file = Object.values(body).find((v): v is File => v instanceof File)
 
 			if (file) {
-				const roles = await validateAndGetRoles(c.req.raw, secret)
-				const isStaff = roles?.some((role) => STAFF_ROLES.has(role)) ?? false
-				const limit = maxUploadBytes(c.env.MAX_UPLOAD_BYTES)
 				if (!isStaff && file.size > limit) {
 					return c.json({ error: `file exceeds the ${limit}-byte upload limit` }, 413)
 				}
@@ -208,6 +368,13 @@ const app = new Hono<App>()
 					// worker answer one from a HEAD instead of downloading the blob to digest it.
 					sha256: await crypto.subtle.digest('SHA-256', bytes),
 				})
+				if (c.req.header('user-agent')?.startsWith('RRStudio')) {
+					const hash = await crypto.subtle.digest('SHA-256', bytes)
+					return c.json({
+						Filename: filename,
+						Hash: btoa(String.fromCharCode(...new Uint8Array(hash))),
+					})
+				}
 				return c.json({ filename })
 			}
 

apps/storage/src/test/integration/api.test.ts

@@ -235,7 +235,7 @@ it('GET /openapi.json documents every route', async () => {
 			Object.keys(ops).map((method) => `${method.toUpperCase()} ${path}`)
 		)
 	)
-	expect([...documented].sort()).toEqual(['GET /', 'POST /upload'])
+	expect([...documented].sort()).toEqual(['GET /', 'POST /upload', 'POST /upload/unityassets'])
 
 	// Every operation carries a summary — a path present but undescribed is not
 	// documentation.
@@ -246,3 +246,167 @@ it('GET /openapi.json documents every route', async () => {
 	// Every schema inlines: a `$ref` here would be a dangling reference (see openapi.ts).
 	expect(JSON.stringify(spec).includes('"$ref"')).toBe(false)
 })
+
+it('uploads Studio source files in order with verified hashes and a manifest under its own prefix', async () => {
+	const headers = { ...(await bearer()), 'User-Agent': 'RRStudio' }
+	const body = new FormData()
+	body.append('file', new File(['first'], 'a.txt'))
+	body.append('file', new File(['second'], 'b.txt'))
+	const res = await SELF.fetch(`${ORIGIN}/upload/unityassets`, { method: 'POST', headers, body })
+	expect(res.status).toBe(200)
+	const files = (await res.json()) as Array<{ Filename: string; Hash: string }>
+	expect(files).toHaveLength(2)
+	for (const [i, file] of files.entries()) {
+		expect(file.Filename).toMatch(/^[0-9a-z]{40}$/)
+		const stored = await env.CDN_ASSETS.get(`unityasset/${file.Filename}`)
+		expect(await stored!.text()).toBe(i === 0 ? 'first' : 'second')
+		const hash = await crypto.subtle.digest(
+			'SHA-256',
+			new TextEncoder().encode(i === 0 ? 'first' : 'second')
+		)
+		expect(file.Hash).toBe(btoa(String.fromCharCode(...new Uint8Array(hash))))
+	}
+	const manifest = new FormData()
+	manifest.append('fileType', '9')
+	manifest.append(
+		'file',
+		new File(['{"Version":3,"Files":[]}'], 'manifest.json', { type: 'application/json' })
+	)
+	const uploaded = await SELF.fetch(`${ORIGIN}/upload`, { method: 'POST', headers, body: manifest })
+	expect(uploaded.status).toBe(200)
+	const source = (await uploaded.json()) as { Filename: string; Hash: string }
+	expect(source.Hash).toBeTruthy()
+	expect(await (await env.CDN_ASSETS.get(`roomunityasset/${source.Filename}`))!.text()).toBe(
+		'{"Version":3,"Files":[]}'
+	)
+	const unauthenticated = new FormData()
+	unauthenticated.append('file', new File(['x'], 'x.txt'))
+	expect(
+		(await SELF.fetch(`${ORIGIN}/upload/unityassets`, { method: 'POST', body: unauthenticated }))
+			.status
+	).toBe(401)
+})
+
+it('accepts the Mono multipart wire format without changing binary file bytes', async () => {
+	const boundary = 'mono-studio-boundary'
+	const payload = new Uint8Array([0, 255, 128, 13, 10, 0, 200])
+	const parts: Array<string | Uint8Array> = []
+	for (let i = 0; i < 2; i++)
+		parts.push(
+			`--${boundary}\r\nContent-Type: application/octet-stream\r\nContent-Disposition: form-data; name=file; filename=file.bin; filename*=utf-8''file.bin\r\n\r\n`,
+			payload,
+			'\r\n'
+		)
+	parts.push(`--${boundary}--\r\n`)
+	const body = await new Response(
+		new Blob(parts).stream().pipeThrough(new CompressionStream('gzip'))
+	).arrayBuffer()
+	const result = await SELF.fetch(`${ORIGIN}/upload/unityassets`, {
+		method: 'POST',
+		headers: {
+			...(await bearer()),
+			'Content-Type': `multipart/form-data; boundary="${boundary}"`,
+			'Content-Encoding': 'gzip',
+		},
+		body,
+	})
+	expect(result.status).toBe(200)
+	const files = await result.json<Array<{ Filename: string }>>()
+	expect(files).toHaveLength(2)
+	expect(files[0].Filename).toBe(files[1].Filename)
+	const saved = await env.CDN_ASSETS.get(`unityasset/${files[0].Filename}`)
+	expect(new Uint8Array(await saved!.arrayBuffer())).toEqual(payload)
+})
+
+it('accepts Studio gzip multipart batches with stable content-addressed filenames', async () => {
+	const body = new FormData()
+	body.append('file', new File(['first'], 'file.bin'))
+	body.append('file', new File(['second'], 'file.bin'))
+	body.append('file', new File(['first'], 'file.bin'))
+	const request = new Request(`${ORIGIN}/upload/unityassets`, { method: 'POST', body })
+	const compressed = request.body!.pipeThrough(new CompressionStream('gzip'))
+	const bytes = await new Response(compressed).arrayBuffer()
+	const headers = {
+		...(await bearer()),
+		'User-Agent': 'RRStudio',
+		'Content-Type': request.headers.get('content-type')!,
+		'Content-Encoding': 'gzip',
+	}
+	const upload = async () => SELF.fetch(request.url, { method: 'POST', headers, body: bytes })
+	const result = await upload()
+	expect(result.status).toBe(200)
+	const files = await result.json<Array<{ Filename: string; Hash: string }>>()
+	expect(files).toHaveLength(3)
+	expect(files.map((file) => file.Filename)).toEqual([
+		'3jz19z865iu7adldcdxv6stlddxq0yp5qjvnon77',
+		'4wk7uj0ueguhyik1bkmvgcl60upxvkg9wwjbdlrk',
+		'3jz19z865iu7adldcdxv6stlddxq0yp5qjvnon77',
+	])
+	expect(files[0]).toEqual(files[2])
+	expect(files[0]!.Filename).not.toBe(files[1]!.Filename)
+	for (const [i, file] of files.entries()) {
+		expect(await (await env.CDN_ASSETS.get(`unityasset/${file.Filename}`))!.text()).toBe(
+			i === 1 ? 'second' : 'first'
+		)
+	}
+	expect(await (await upload()).json()).toEqual(files)
+	// The next SDK request compresses the source manifest through /upload too.
+	const manifest = JSON.stringify({
+		Version: 3,
+		Files: files.map((file, i) => ({ n: file.Filename, p: `Assets/${i}.txt` })),
+	})
+	const manifestForm = new FormData()
+	manifestForm.append('fileType', '9')
+	manifestForm.append('file', new File([manifest], 'file.bin', { type: 'application/json' }))
+	const manifestRequest = new Request(`${ORIGIN}/upload`, { method: 'POST', body: manifestForm })
+	const manifestBytes = await new Response(
+		manifestRequest.body!.pipeThrough(new CompressionStream('gzip'))
+	).arrayBuffer()
+	const saved = await SELF.fetch(manifestRequest.url, {
+		method: 'POST',
+		headers: { ...headers, 'Content-Type': manifestRequest.headers.get('content-type')! },
+		body: manifestBytes,
+	})
+	expect(saved.status).toBe(200)
+	const metadata = await saved.json<{ Filename: string }>()
+	expect(await (await env.CDN_ASSETS.get(`roomunityasset/${metadata.Filename}`))!.text()).toBe(
+		manifest
+	)
+})
+
+it('rejects malformed and oversized compressed source uploads without an internal error', async () => {
+	const headers = {
+		...(await bearer()),
+		'Content-Encoding': 'gzip',
+		'Content-Type': 'multipart/form-data; boundary=test',
+	}
+	expect(
+		(
+			await SELF.fetch(`${ORIGIN}/upload/unityassets`, {
+				method: 'POST',
+				headers,
+				body: 'invalid gzip',
+			})
+		).status
+	).toBe(400)
+	const original = env.MAX_UPLOAD_BYTES
+	env.MAX_UPLOAD_BYTES = '16'
+	try {
+		const form = new FormData()
+		form.append('file', new File([new Uint8Array(100_000)], 'large.bin'))
+		const request = new Request(`${ORIGIN}/upload/unityassets`, { method: 'POST', body: form })
+		const compressed = await new Response(
+			request.body!.pipeThrough(new CompressionStream('gzip'))
+		).arrayBuffer()
+		expect(compressed.byteLength).toBeLessThan(1000)
+		const response = await SELF.fetch(request.url, {
+			method: 'POST',
+			headers: { ...headers, 'Content-Type': request.headers.get('content-type')! },
+			body: compressed,
+		})
+		expect(response.status).toBe(413)
+	} finally {
+		if (original === undefined) delete env.MAX_UPLOAD_BYTES
+		else env.MAX_UPLOAD_BYTES = original
+	}
+})