From c1000c82aa7aed08bd139648ee168ecd63e206ba Mon Sep 17 00:00:00 2001 From: Alexa <53471801+Alexa-RR@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:32:09 +0200 Subject: [PATCH 1/3] fix: let EF own the Postgres bootstrap transaction (EF Core 10) EF Core 10 runs EnsureCreated through the migration executor, which begins its own transaction unless EF already has one. The bootstrap held the advisory lock in a transaction begun on the raw connection, invisible to EF, so a fresh Postgres database failed at startup with "A transaction is already in progress". Begin it through db.Database and take the lock with ExecuteSqlRaw so EnsureCreated runs inside it. --- DorkNet.Server/Startup/DatabaseBootstrap.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/DorkNet.Server/Startup/DatabaseBootstrap.cs b/DorkNet.Server/Startup/DatabaseBootstrap.cs index 485b5bc..bb5b946 100644 --- a/DorkNet.Server/Startup/DatabaseBootstrap.cs +++ b/DorkNet.Server/Startup/DatabaseBootstrap.cs @@ -82,13 +82,14 @@ public static class DatabaseBootstrap await Task.Delay(delay); } } - await using var tx = await conn.BeginTransactionAsync(); - await using (var lockCmd = conn.CreateCommand()) - { - lockCmd.Transaction = tx; - lockCmd.CommandText = "SELECT pg_advisory_xact_lock(1146246987);"; - await lockCmd.ExecuteNonQueryAsync(); - } + // Begin the transaction through EF, not on the raw connection: + // since EF Core 10, EnsureCreated runs its commands through the + // migration executor, which starts its own transaction unless EF + // already has one, and fails on a fresh database with "A + // transaction is already in progress" when the open one is + // invisible to it. + await using var tx = await db.Database.BeginTransactionAsync(); + await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_xact_lock(1146246987);"); db.Database.EnsureCreated(); await tx.CommitAsync(); From bb22675a914457a875e55a95f4555382f4434175 Mon Sep 17 00:00:00 2001 From: Alexa <53471801+Alexa-RR@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:01:43 +0200 Subject: [PATCH 2/3] fix: hold the Postgres bootstrap lock across schema creation and seeding Booting several replicas against one fresh Postgres database left all but one dead on startup: the advisory lock only covered EnsureCreated, so the seeders (system account, rooms, playlists, store, ...) raced and hit duplicate keys. Take a session-level advisory lock for the whole bootstrap and release it explicitly (pooled connections would otherwise keep it). The upgrades and seeds both providers run move into RunUpgradesAndSeedsAsync. --- DorkNet.Server/Startup/DatabaseBootstrap.cs | 124 ++++++++++++-------- 1 file changed, 73 insertions(+), 51 deletions(-) diff --git a/DorkNet.Server/Startup/DatabaseBootstrap.cs b/DorkNet.Server/Startup/DatabaseBootstrap.cs index bb5b946..13489a7 100644 --- a/DorkNet.Server/Startup/DatabaseBootstrap.cs +++ b/DorkNet.Server/Startup/DatabaseBootstrap.cs @@ -15,9 +15,9 @@ namespace DorkNet.Server.Startup; /// SQLite — Database.Migrate() after /// reconciles any legacy /// migration-history state. -/// Postgres — EnsureCreated behind a transaction-scoped -/// advisory lock so concurrent replica boots don't race on -/// CREATE TABLE. Connect retries cover the +/// Postgres — EnsureCreated, with the whole bootstrap +/// (schema and seeds) behind a session advisory lock so concurrent +/// replica boots take turns instead of racing. Connect retries cover the /// container-orchestrated boot window where Postgres DNS isn't /// resolvable yet. /// @@ -40,62 +40,84 @@ public static class DatabaseBootstrap await ConfigureSqliteForLocalConcurrencyAsync(db); BaselineExistingSchemaIfNeeded(db); db.Database.Migrate(); + await RunUpgradesAndSeedsAsync(app, scope, db); + return; } - else + + // Postgres path: schema generated from the EF Core model via + // EnsureCreated, then the same upgrades and seeds as SQLite. The + // whole bootstrap holds a session-level advisory lock, so replicas + // booting against one database take turns: the first creates the + // schema and seeds, the rest find everything in place. Holding it + // only around EnsureCreated left the seeders racing (duplicate keys + // on a fresh database). + // + // Lock key 0x444F524B (ascii "DORK") is hard-coded; it just needs + // to be the same int64 across replicas and not collide with + // anything else in the database. It is released in the finally + // below, and Postgres drops it if the replica's connection dies, so + // a crashed replica can't wedge the next boot. + var conn = db.Database.GetDbConnection(); + + // Retry initial Postgres connect — on container-orchestrated boots + // (Coolify, docker-compose) the Postgres service's DNS often isn't + // resolvable yet when this app starts, surfacing as SocketException + // errno=11 EAGAIN from Npgsql's Dns.GetHostEntryOrAddressesCore. + // Postgres itself can also be up-but-not-accepting-connections for + // a few seconds while it replays WAL. 15 attempts × ~2s backoff + // covers the typical 5-30s warmup window without looping forever + // on a permanently broken config. + var connectLogger = app.Services.GetRequiredService() + .CreateLogger("DorkNet.Server.Bootstrap"); + for (var attempt = 1; ; attempt++) { - // Postgres path: schema generated from the EF Core model via - // EnsureCreated. Wrapped in a transaction-scoped advisory lock - // so concurrent replicas booting against the same database - // don't race on CREATE TABLE — only the first replica to take - // the lock runs EnsureCreated, the rest see "schema already - // exists" and proceed. - // - // Lock key 0x444F524B (ascii "DORK") is hard-coded; it just - // needs to be the same int64 across replicas and not collide - // with anything else in the database. pg_advisory_xact_lock - // releases automatically on COMMIT, so a crashed replica can't - // wedge the next boot. - var conn = db.Database.GetDbConnection(); - - // Retry initial Postgres connect — on container-orchestrated boots - // (Coolify, docker-compose) the Postgres service's DNS often isn't - // resolvable yet when this app starts, surfacing as SocketException - // errno=11 EAGAIN from Npgsql's Dns.GetHostEntryOrAddressesCore. - // Postgres itself can also be up-but-not-accepting-connections for - // a few seconds while it replays WAL. 15 attempts × ~2s backoff - // covers the typical 5-30s warmup window without looping forever - // on a permanently broken config. - var connectLogger = app.Services.GetRequiredService() - .CreateLogger("DorkNet.Server.Bootstrap"); - for (var attempt = 1; ; attempt++) + try { await conn.OpenAsync(); break; } + catch (Exception ex) when (attempt < 15 && + (ex is System.Net.Sockets.SocketException + || ex is Npgsql.NpgsqlException + || ex is System.Data.Common.DbException)) { - try { await conn.OpenAsync(); break; } - catch (Exception ex) when (attempt < 15 && - (ex is System.Net.Sockets.SocketException - || ex is Npgsql.NpgsqlException - || ex is System.Data.Common.DbException)) - { - var delay = TimeSpan.FromSeconds(Math.Min(attempt, 5)); - connectLogger.LogWarning( - "[bootstrap] Postgres connect attempt {Attempt}/15 failed ({Type}: {Message}); retrying in {Delay}s", - attempt, ex.GetType().Name, ex.Message, delay.TotalSeconds); - await Task.Delay(delay); - } + var delay = TimeSpan.FromSeconds(Math.Min(attempt, 5)); + connectLogger.LogWarning( + "[bootstrap] Postgres connect attempt {Attempt}/15 failed ({Type}: {Message}); retrying in {Delay}s", + attempt, ex.GetType().Name, ex.Message, delay.TotalSeconds); + await Task.Delay(delay); } - // Begin the transaction through EF, not on the raw connection: - // since EF Core 10, EnsureCreated runs its commands through the - // migration executor, which starts its own transaction unless EF - // already has one, and fails on a fresh database with "A - // transaction is already in progress" when the open one is - // invisible to it. - await using var tx = await db.Database.BeginTransactionAsync(); - await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_xact_lock(1146246987);"); - db.Database.EnsureCreated(); - await tx.CommitAsync(); + } + // The connection stays open until the finally, so every step below + // (they all share this scope's DbContext) runs on the session that + // holds the lock. + await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_lock(1146246987);"); + try + { + // Begin the transaction through EF, not on the raw connection: + // EnsureCreated runs its commands through the migration executor, + // which starts its own transaction unless EF already has one, and + // fails on a fresh database with "A transaction is already in + // progress" when the open one is invisible to it. + await using (var tx = await db.Database.BeginTransactionAsync()) + { + db.Database.EnsureCreated(); + await tx.CommitAsync(); + } + await RunUpgradesAndSeedsAsync(app, scope, db); + } + finally + { + // Explicit: Npgsql pools connections, so closing alone would not + // end the session that holds the lock. + await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_unlock(1146246987);"); await conn.CloseAsync(); } + } + /// The idempotent post-schema sequence both providers run: + /// column/table upgrades, system account and backfills, legacy data + /// transforms, canonical rooms and the per-feature seeders. Flips the + /// /healthz readiness flag at the end. + private static async Task RunUpgradesAndSeedsAsync(WebApplication app, IServiceScope scope, DorkNetDbContext db) + { // Signup-code + pending-device tables. These post-date the // consolidated Initial migration, and the Postgres path is // EnsureCreated-only (no migration replay), so neither provider From d0d26c9821da5a05bffc53b90ddc172ba0590578 Mon Sep 17 00:00:00 2001 From: Alexa <53471801+Alexa-RR@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:52:20 +0200 Subject: [PATCH 3/3] fix: wait for the bootstrap lock without a command timeout; safe unlock From the local CodeRabbit review: - A replica waiting for the lock used the default 30 s command timeout and failed to boot while another replica seeded a fresh database. Wait with no timeout, logging that it is waiting. - An unlock failure in the finally (e.g. the connection already broken) replaced the real bootstrap error and skipped closing the connection. Log it instead, and always close. --- DorkNet.Server/Startup/DatabaseBootstrap.cs | 36 ++++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/DorkNet.Server/Startup/DatabaseBootstrap.cs b/DorkNet.Server/Startup/DatabaseBootstrap.cs index 13489a7..5ac885c 100644 --- a/DorkNet.Server/Startup/DatabaseBootstrap.cs +++ b/DorkNet.Server/Startup/DatabaseBootstrap.cs @@ -87,8 +87,21 @@ public static class DatabaseBootstrap // The connection stays open until the finally, so every step below // (they all share this scope's DbContext) runs on the session that - // holds the lock. - await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_lock(1146246987);"); + // holds the lock. Waiting for it has no command timeout: another + // replica may be seeding a fresh database (thumbnail downloads + // included) for well over the default 30 s. + connectLogger.LogInformation("[bootstrap] waiting for the database bootstrap lock"); + var previousTimeout = db.Database.GetCommandTimeout(); + db.Database.SetCommandTimeout(0); + try + { + await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_lock(1146246987);"); + } + finally + { + db.Database.SetCommandTimeout(previousTimeout); + } + connectLogger.LogInformation("[bootstrap] holding the database bootstrap lock"); try { // Begin the transaction through EF, not on the raw connection: @@ -106,9 +119,22 @@ public static class DatabaseBootstrap finally { // Explicit: Npgsql pools connections, so closing alone would not - // end the session that holds the lock. - await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_unlock(1146246987);"); - await conn.CloseAsync(); + // end the session that holds the lock. If the connection is + // already broken the unlock fails too; that must not replace the + // real bootstrap error, and the connection still gets closed + // (Postgres then drops the lock with the session). + try + { + await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_unlock(1146246987);"); + } + catch (Exception ex) + { + connectLogger.LogWarning(ex, "[bootstrap] releasing the database bootstrap lock failed"); + } + finally + { + await conn.CloseAsync(); + } } }