using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using DorkNet.Server.Auth; using DorkNet.Server.Data; using DorkNet.Server.Services; namespace DorkNet.Server.Controllers.Admin; /// /// admin.*/api/admin/v1/storage/backfill — one-shot migrator that /// moves the legacy RoomDataBlobs.Bytes column out of the /// database and into S3 at the canonical key from /// . After every row's bytes have been /// uploaded and verified, a follow-up EF migration drops the /// Bytes column entirely. /// /// Why this exists: every controller that writes new content already /// targets S3 directly (the importer, photo upload, profile image, /// dorm save, room save, …). The Bytes column only holds /// pre-cutover data — rooms imported before the S3-only refactor /// landed. This endpoint drains that legacy column into S3 once and /// then we never look at it again. /// /// Runs as a background task so the HTTP request returns immediately /// even for a multi-GB drain. Progress streams to the server log and /// to a small in-memory snapshot the UI /// can poll. Idempotent — each row is checked against /// first, and rows whose /// bytes are already in S3 get their DB Bytes cleared without /// re-upload. /// [ApiController] [Route("api/admin/v1/storage")] [Authorize] [AdminOnly] public class StorageBackfillController( ILogger log, IServiceScopeFactory scopeFactory) : ControllerBase { private static readonly object _gate = new(); private static BackfillStatus _status = BackfillStatus.Idle; /// POST /storage/backfill?dryRun=true — start a /// drain pass. Returns 200 immediately; the actual work happens /// in a background task. Re-POSTing while running returns 409. /// Set ?dryRun=true on the first pass to log every /// (row → bucket/key) decision without writing to S3 or clearing /// the DB Bytes column. [HttpPost("backfill")] public IActionResult Start([FromQuery] bool dryRun = false) { lock (_gate) { if (_status.Running) return Conflict(new { error = "already_running", since = _status.StartedAt }); _status = BackfillStatus.Starting(dryRun); } _ = Task.Run(() => RunAsync(dryRun)); return Ok(new { started = true, dryRun }); } /// GET /storage/backfill/status — current /// progress. Updated row-by-row during a run. [HttpGet("backfill/status")] public IActionResult Status() => Ok(_status); private async Task RunAsync(bool dryRun) { var startedAt = DateTime.UtcNow; try { // Phase 1: pull the list of row ids that still have bytes. // Materialise just the ids (not the bytes) up-front so we // don't hold a reader open across per-row S3 PUTs — that // would conflict with the per-row ExecuteUpdateAsync on // Npgsql, which doesn't allow a second command while a // reader is mid-iteration. Bytes get fetched one row at a // time below so RAM only holds one blob at a time. // Filter on Bytes != null only — Npgsql can't translate // .Length on a byte[] property (it sees it as Enumerable.Any // and bails). Empty-byte-array rows don't exist in our // schema; rows either hold real bytes or NULL after backfill. // We still re-check Length on the materialised row before // uploading, defensive against any future client-evaluated // edge case. List rowIds; long total; await using (var scope = scopeFactory.CreateAsyncScope()) { var dbScoped = scope.ServiceProvider.GetRequiredService(); rowIds = await dbScoped.RoomDataBlobs .AsNoTracking() .Where(b => b.Bytes != null) .OrderBy(b => b.Id) .Select(b => b.Id) .ToListAsync(); total = rowIds.Count; } log.LogInformation( "[backfill] starting — rowsWithBytes={Total} dryRun={DryRun}", total, dryRun); lock (_gate) _status = _status with { Total = total }; // Phase 2: per-row upload + clear. Each row gets its own // short-lived DbContext so the reader/writer conflict is // structurally impossible. Memory-bound at one row's // bytes plus query overhead. foreach (var rowId in rowIds) { await using var rowScope = scopeFactory.CreateAsyncScope(); var rowDb = rowScope.ServiceProvider.GetRequiredService(); var storageScoped = rowScope.ServiceProvider.GetRequiredService(); var row = await rowDb.RoomDataBlobs .AsNoTracking() .Where(b => b.Id == rowId) .Select(b => new { b.Id, b.BlobName, b.Bytes }) .FirstOrDefaultAsync(); if (row is null || row.Bytes is null || row.Bytes.Length == 0) { Bump(s => s with { Skipped = s.Skipped + 1 }); continue; } var (bucket, key) = BlobRouter.Route(row.BlobName); try { if (dryRun) { log.LogInformation( "[backfill:dry] {Blob} → {Bucket}/{Key} ({Bytes} B)", row.BlobName, bucket, key, row.Bytes.Length); Bump(s => s with { Uploaded = s.Uploaded + 1, LastBlobName = row.BlobName }); continue; } // Upload unconditionally (S3 PUT is idempotent / // overwriting). Verifying with ExistsAsync first // would let us skip already-mirrored rows, but a // mismatch between S3 and DB bytes (corrupted // earlier upload, partial backfill, etc.) would // leave the divergence in place. Re-uploading // guarantees S3 == DB before we clear DB. var contentType = MimeFromName(row.BlobName); // Per-row timeout: 4 minutes generous enough for a // 100 MB blob over a 5 Mbps origin link, well // inside the 5-minute S3 SDK ceiling. Without a // CT, a hung S3 connection could freeze the whole // drain indefinitely. using var putCt = new CancellationTokenSource(TimeSpan.FromMinutes(4)); await storageScoped.PutAsync(bucket, key, row.Bytes, contentType, putCt.Token); // SAFETY GATE: confirm S3 actually persisted the // object before dropping the DB copy. PutAsync // returning success isn't enough — some S3 impls // can ack a write that disappears (eventual // consistency, replication lag, write buffer // dropped on shutdown). HEAD costs nothing. using var headCt = new CancellationTokenSource(TimeSpan.FromSeconds(15)); if (!await storageScoped.ExistsAsync(bucket, key, headCt.Token)) throw new InvalidOperationException( $"S3 PUT {bucket}/{key} returned success but ExistsAsync says no — refusing to clear DB bytes"); await rowDb.RoomDataBlobs .Where(b => b.Id == row.Id) .ExecuteUpdateAsync(s => s.SetProperty(b => b.Bytes, (byte[]?)null)); Bump(s => s with { Uploaded = s.Uploaded + 1, LastBlobName = row.BlobName }); } catch (Exception ex) { log.LogWarning(ex, "[backfill] failed {Blob} → {Bucket}/{Key} — DB bytes left intact", row.BlobName, bucket, key); Bump(s => s with { Failed = s.Failed + 1, LastBlobName = row.BlobName }); } } var elapsed = (int)(DateTime.UtcNow - startedAt).TotalMilliseconds; lock (_gate) _status = _status with { Running = false, FinishedAt = DateTime.UtcNow, ElapsedMs = elapsed, }; log.LogInformation( "[backfill] finished — total={Total} uploaded={Up} alreadyMirrored={Already} skipped={Skip} failed={Fail} dryRun={Dry} elapsed={Ms}ms", _status.Total, _status.Uploaded, _status.AlreadyMirrored, _status.Skipped, _status.Failed, dryRun, elapsed); } catch (Exception ex) { log.LogError(ex, "[backfill] crashed"); lock (_gate) _status = _status with { Running = false, FinishedAt = DateTime.UtcNow, Error = ex.Message }; } } private static void Bump(Func mutate) { lock (_gate) _status = mutate(_status); } private static string MimeFromName(string name) => Path.GetExtension(name).ToLowerInvariant() switch { ".png" => "image/png", ".jpg" or ".jpeg" => "image/jpeg", ".gif" => "image/gif", ".webp" => "image/webp", ".mp4" => "video/mp4", _ => "application/octet-stream", }; public sealed record BackfillStatus( bool Running, bool DryRun, DateTime? StartedAt, DateTime? FinishedAt, long Total, long Uploaded, long AlreadyMirrored, long Skipped, long Failed, string? LastBlobName, int? ElapsedMs, string? Error) { public static BackfillStatus Idle => new(false, false, null, null, 0, 0, 0, 0, 0, null, null, null); public static BackfillStatus Starting(bool dryRun) => new(true, dryRun, DateTime.UtcNow, null, 0, 0, 0, 0, 0, null, null, null); } }