Compare commits

...

3 Commits

Author SHA1 Message Date
Alexa d0d26c9821 fix: wait for the bootstrap lock without a command timeout; safe unlock
From the local CodeRabbit review:
- A replica waiting for the lock used the default 30 s command timeout and
  failed to boot while another replica seeded a fresh database. Wait with
  no timeout, logging that it is waiting.
- An unlock failure in the finally (e.g. the connection already broken)
  replaced the real bootstrap error and skipped closing the connection.
  Log it instead, and always close.
2026-10-25 13:52:20 +02:00
Alexa bb22675a91 fix: hold the Postgres bootstrap lock across schema creation and seeding
Booting several replicas against one fresh Postgres database left all but
one dead on startup: the advisory lock only covered EnsureCreated, so the
seeders (system account, rooms, playlists, store, ...) raced and hit
duplicate keys. Take a session-level advisory lock for the whole bootstrap
and release it explicitly (pooled connections would otherwise keep it).
The upgrades and seeds both providers run move into
RunUpgradesAndSeedsAsync.
2026-10-25 13:01:43 +02:00
Alexa c1000c82aa fix: let EF own the Postgres bootstrap transaction (EF Core 10)
EF Core 10 runs EnsureCreated through the migration executor, which begins
its own transaction unless EF already has one. The bootstrap held the
advisory lock in a transaction begun on the raw connection, invisible to
EF, so a fresh Postgres database failed at startup with "A transaction is
already in progress". Begin it through db.Database and take the lock with
ExecuteSqlRaw so EnsureCreated runs inside it.
2026-10-25 13:00:16 +02:00
+70 -21
View File
@@ -15,9 +15,9 @@ namespace DorkNet.Server.Startup;
/// <item>SQLite — <c>Database.Migrate()</c> after /// <item>SQLite — <c>Database.Migrate()</c> after
/// <see cref="BaselineExistingSchemaIfNeeded"/> reconciles any legacy /// <see cref="BaselineExistingSchemaIfNeeded"/> reconciles any legacy
/// migration-history state.</item> /// migration-history state.</item>
/// <item>Postgres — <c>EnsureCreated</c> behind a transaction-scoped /// <item>Postgres — <c>EnsureCreated</c>, with the whole bootstrap
/// advisory lock so concurrent replica boots don't race on /// (schema and seeds) behind a session advisory lock so concurrent
/// <c>CREATE TABLE</c>. Connect retries cover the /// replica boots take turns instead of racing. Connect retries cover the
/// container-orchestrated boot window where Postgres DNS isn't /// container-orchestrated boot window where Postgres DNS isn't
/// resolvable yet.</item> /// resolvable yet.</item>
/// </list> /// </list>
@@ -40,21 +40,23 @@ public static class DatabaseBootstrap
await ConfigureSqliteForLocalConcurrencyAsync(db); await ConfigureSqliteForLocalConcurrencyAsync(db);
BaselineExistingSchemaIfNeeded(db); BaselineExistingSchemaIfNeeded(db);
db.Database.Migrate(); db.Database.Migrate();
await RunUpgradesAndSeedsAsync(app, scope, db);
return;
} }
else
{
// Postgres path: schema generated from the EF Core model via // Postgres path: schema generated from the EF Core model via
// EnsureCreated. Wrapped in a transaction-scoped advisory lock // EnsureCreated, then the same upgrades and seeds as SQLite. The
// so concurrent replicas booting against the same database // whole bootstrap holds a session-level advisory lock, so replicas
// don't race on CREATE TABLE — only the first replica to take // booting against one database take turns: the first creates the
// the lock runs EnsureCreated, the rest see "schema already // schema and seeds, the rest find everything in place. Holding it
// exists" and proceed. // only around EnsureCreated left the seeders racing (duplicate keys
// on a fresh database).
// //
// Lock key 0x444F524B (ascii "DORK") is hard-coded; it just // Lock key 0x444F524B (ascii "DORK") is hard-coded; it just needs
// needs to be the same int64 across replicas and not collide // to be the same int64 across replicas and not collide with
// with anything else in the database. pg_advisory_xact_lock // anything else in the database. It is released in the finally
// releases automatically on COMMIT, so a crashed replica can't // below, and Postgres drops it if the replica's connection dies, so
// wedge the next boot. // a crashed replica can't wedge the next boot.
var conn = db.Database.GetDbConnection(); var conn = db.Database.GetDbConnection();
// Retry initial Postgres connect — on container-orchestrated boots // Retry initial Postgres connect — on container-orchestrated boots
@@ -82,19 +84,66 @@ public static class DatabaseBootstrap
await Task.Delay(delay); await Task.Delay(delay);
} }
} }
await using var tx = await conn.BeginTransactionAsync();
await using (var lockCmd = conn.CreateCommand()) // The connection stays open until the finally, so every step below
// (they all share this scope's DbContext) runs on the session that
// holds the lock. Waiting for it has no command timeout: another
// replica may be seeding a fresh database (thumbnail downloads
// included) for well over the default 30 s.
connectLogger.LogInformation("[bootstrap] waiting for the database bootstrap lock");
var previousTimeout = db.Database.GetCommandTimeout();
db.Database.SetCommandTimeout(0);
try
{ {
lockCmd.Transaction = tx; await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_lock(1146246987);");
lockCmd.CommandText = "SELECT pg_advisory_xact_lock(1146246987);";
await lockCmd.ExecuteNonQueryAsync();
} }
finally
{
db.Database.SetCommandTimeout(previousTimeout);
}
connectLogger.LogInformation("[bootstrap] holding the database bootstrap lock");
try
{
// Begin the transaction through EF, not on the raw connection:
// EnsureCreated runs its commands through the migration executor,
// which starts its own transaction unless EF already has one, and
// fails on a fresh database with "A transaction is already in
// progress" when the open one is invisible to it.
await using (var tx = await db.Database.BeginTransactionAsync())
{
db.Database.EnsureCreated(); db.Database.EnsureCreated();
await tx.CommitAsync(); await tx.CommitAsync();
}
await RunUpgradesAndSeedsAsync(app, scope, db);
}
finally
{
// Explicit: Npgsql pools connections, so closing alone would not
// end the session that holds the lock. If the connection is
// already broken the unlock fails too; that must not replace the
// real bootstrap error, and the connection still gets closed
// (Postgres then drops the lock with the session).
try
{
await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_unlock(1146246987);");
}
catch (Exception ex)
{
connectLogger.LogWarning(ex, "[bootstrap] releasing the database bootstrap lock failed");
}
finally
{
await conn.CloseAsync(); await conn.CloseAsync();
} }
}
}
/// <summary>The idempotent post-schema sequence both providers run:
/// column/table upgrades, system account and backfills, legacy data
/// transforms, canonical rooms and the per-feature seeders. Flips the
/// <c>/healthz</c> readiness flag at the end.</summary>
private static async Task RunUpgradesAndSeedsAsync(WebApplication app, IServiceScope scope, DorkNetDbContext db)
{
// Signup-code + pending-device tables. These post-date the // Signup-code + pending-device tables. These post-date the
// consolidated Initial migration, and the Postgres path is // consolidated Initial migration, and the Postgres path is
// EnsureCreated-only (no migration replay), so neither provider // EnsureCreated-only (no migration replay), so neither provider