fix: wait for the bootstrap lock without a command timeout; safe unlock

From the local CodeRabbit review:
- A replica waiting for the lock used the default 30 s command timeout and
  failed to boot while another replica seeded a fresh database. Wait with
  no timeout, logging that it is waiting.
- An unlock failure in the finally (e.g. the connection already broken)
  replaced the real bootstrap error and skipped closing the connection.
  Log it instead, and always close.

DorkNet.Server/Startup/DatabaseBootstrap.cs

@@ -15,9 +15,9 @@ namespace DorkNet.Server.Startup;
 ///   <item>SQLite — <c>Database.Migrate()</c> after
 ///   <see cref="BaselineExistingSchemaIfNeeded"/> reconciles any legacy
 ///   migration-history state.</item>
-///   <item>Postgres — <c>EnsureCreated</c> behind a transaction-scoped
-///   advisory lock so concurrent replica boots don't race on
-///   <c>CREATE TABLE</c>. Connect retries cover the
+///   <item>Postgres — <c>EnsureCreated</c>, with the whole bootstrap
+///   (schema and seeds) behind a session advisory lock so concurrent
+///   replica boots take turns instead of racing. Connect retries cover the
 ///   container-orchestrated boot window where Postgres DNS isn't
 ///   resolvable yet.</item>
 /// </list>
@@ -40,61 +40,110 @@ public static class DatabaseBootstrap
             await ConfigureSqliteForLocalConcurrencyAsync(db);
             BaselineExistingSchemaIfNeeded(db);
             db.Database.Migrate();
+            await RunUpgradesAndSeedsAsync(app, scope, db);
+            return;
         }
-        else
+
+        // Postgres path: schema generated from the EF Core model via
+        // EnsureCreated, then the same upgrades and seeds as SQLite. The
+        // whole bootstrap holds a session-level advisory lock, so replicas
+        // booting against one database take turns: the first creates the
+        // schema and seeds, the rest find everything in place. Holding it
+        // only around EnsureCreated left the seeders racing (duplicate keys
+        // on a fresh database).
+        //
+        // Lock key 0x444F524B (ascii "DORK") is hard-coded; it just needs
+        // to be the same int64 across replicas and not collide with
+        // anything else in the database. It is released in the finally
+        // below, and Postgres drops it if the replica's connection dies, so
+        // a crashed replica can't wedge the next boot.
+        var conn = db.Database.GetDbConnection();
+
+        // Retry initial Postgres connect — on container-orchestrated boots
+        // (Coolify, docker-compose) the Postgres service's DNS often isn't
+        // resolvable yet when this app starts, surfacing as SocketException
+        // errno=11 EAGAIN from Npgsql's Dns.GetHostEntryOrAddressesCore.
+        // Postgres itself can also be up-but-not-accepting-connections for
+        // a few seconds while it replays WAL. 15 attempts × ~2s backoff
+        // covers the typical 5-30s warmup window without looping forever
+        // on a permanently broken config.
+        var connectLogger = app.Services.GetRequiredService<ILoggerFactory>()
+            .CreateLogger("DorkNet.Server.Bootstrap");
+        for (var attempt = 1; ; attempt++)
         {
-            // Postgres path: schema generated from the EF Core model via
-            // EnsureCreated. Wrapped in a transaction-scoped advisory lock
-            // so concurrent replicas booting against the same database
-            // don't race on CREATE TABLE — only the first replica to take
-            // the lock runs EnsureCreated, the rest see "schema already
-            // exists" and proceed.
-            //
-            // Lock key 0x444F524B (ascii "DORK") is hard-coded; it just
-            // needs to be the same int64 across replicas and not collide
-            // with anything else in the database. pg_advisory_xact_lock
-            // releases automatically on COMMIT, so a crashed replica can't
-            // wedge the next boot.
-            var conn = db.Database.GetDbConnection();
-
-            // Retry initial Postgres connect — on container-orchestrated boots
-            // (Coolify, docker-compose) the Postgres service's DNS often isn't
-            // resolvable yet when this app starts, surfacing as SocketException
-            // errno=11 EAGAIN from Npgsql's Dns.GetHostEntryOrAddressesCore.
-            // Postgres itself can also be up-but-not-accepting-connections for
-            // a few seconds while it replays WAL. 15 attempts × ~2s backoff
-            // covers the typical 5-30s warmup window without looping forever
-            // on a permanently broken config.
-            var connectLogger = app.Services.GetRequiredService<ILoggerFactory>()
-                .CreateLogger("DorkNet.Server.Bootstrap");
-            for (var attempt = 1; ; attempt++)
+            try { await conn.OpenAsync(); break; }
+            catch (Exception ex) when (attempt < 15 &&
+                (ex is System.Net.Sockets.SocketException
+                 || ex is Npgsql.NpgsqlException
+                 || ex is System.Data.Common.DbException))
             {
-                try { await conn.OpenAsync(); break; }
-                catch (Exception ex) when (attempt < 15 &&
-                    (ex is System.Net.Sockets.SocketException
-                     || ex is Npgsql.NpgsqlException
-                     || ex is System.Data.Common.DbException))
-                {
-                    var delay = TimeSpan.FromSeconds(Math.Min(attempt, 5));
-                    connectLogger.LogWarning(
-                        "[bootstrap] Postgres connect attempt {Attempt}/15 failed ({Type}: {Message}); retrying in {Delay}s",
-                        attempt, ex.GetType().Name, ex.Message, delay.TotalSeconds);
-                    await Task.Delay(delay);
-                }
+                var delay = TimeSpan.FromSeconds(Math.Min(attempt, 5));
+                connectLogger.LogWarning(
+                    "[bootstrap] Postgres connect attempt {Attempt}/15 failed ({Type}: {Message}); retrying in {Delay}s",
+                    attempt, ex.GetType().Name, ex.Message, delay.TotalSeconds);
+                await Task.Delay(delay);
             }
-            await using var tx = await conn.BeginTransactionAsync();
-            await using (var lockCmd = conn.CreateCommand())
-            {
-                lockCmd.Transaction = tx;
-                lockCmd.CommandText = "SELECT pg_advisory_xact_lock(1146246987);";
-                await lockCmd.ExecuteNonQueryAsync();
-            }
-            db.Database.EnsureCreated();
-            await tx.CommitAsync();
-
-            await conn.CloseAsync();
         }
 
+        // The connection stays open until the finally, so every step below
+        // (they all share this scope's DbContext) runs on the session that
+        // holds the lock. Waiting for it has no command timeout: another
+        // replica may be seeding a fresh database (thumbnail downloads
+        // included) for well over the default 30 s.
+        connectLogger.LogInformation("[bootstrap] waiting for the database bootstrap lock");
+        var previousTimeout = db.Database.GetCommandTimeout();
+        db.Database.SetCommandTimeout(0);
+        try
+        {
+            await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_lock(1146246987);");
+        }
+        finally
+        {
+            db.Database.SetCommandTimeout(previousTimeout);
+        }
+        connectLogger.LogInformation("[bootstrap] holding the database bootstrap lock");
+        try
+        {
+            // Begin the transaction through EF, not on the raw connection:
+            // EnsureCreated runs its commands through the migration executor,
+            // which starts its own transaction unless EF already has one, and
+            // fails on a fresh database with "A transaction is already in
+            // progress" when the open one is invisible to it.
+            await using (var tx = await db.Database.BeginTransactionAsync())
+            {
+                db.Database.EnsureCreated();
+                await tx.CommitAsync();
+            }
+            await RunUpgradesAndSeedsAsync(app, scope, db);
+        }
+        finally
+        {
+            // Explicit: Npgsql pools connections, so closing alone would not
+            // end the session that holds the lock. If the connection is
+            // already broken the unlock fails too; that must not replace the
+            // real bootstrap error, and the connection still gets closed
+            // (Postgres then drops the lock with the session).
+            try
+            {
+                await db.Database.ExecuteSqlRawAsync("SELECT pg_advisory_unlock(1146246987);");
+            }
+            catch (Exception ex)
+            {
+                connectLogger.LogWarning(ex, "[bootstrap] releasing the database bootstrap lock failed");
+            }
+            finally
+            {
+                await conn.CloseAsync();
+            }
+        }
+    }
+
+    /// <summary>The idempotent post-schema sequence both providers run:
+    /// column/table upgrades, system account and backfills, legacy data
+    /// transforms, canonical rooms and the per-feature seeders. Flips the
+    /// <c>/healthz</c> readiness flag at the end.</summary>
+    private static async Task RunUpgradesAndSeedsAsync(WebApplication app, IServiceScope scope, DorkNetDbContext db)
+    {
         // Signup-code + pending-device tables. These post-date the
         // consolidated Initial migration, and the Postgres path is
         // EnsureCreated-only (no migration replay), so neither provider

Jenkinsfile

@@ -9,9 +9,8 @@
 // from the agent's environment (ARCHIVE_PC_REPO / ARCHIVE_QUEST_REPO), each
 // overridable per run. A build is picked by snapshot tags.
 //
-// A push builds only the platforms whose client code changed (never for
-// docs or CI-only changes). A manual run picks the platforms with PLATFORMS.
-// See ci/jenkins/README.md.
+// On a push, only the platforms whose code changed are rebuilt. A manual run
+// picks the platforms with PLATFORMS. See ci/jenkins/README.md.
 //
 // Finished builds are renamed for players and published to the downloads bucket
 // (ci/downloads) unless PUBLISH_DOWNLOADS is off. The build scripts keep their
@@ -77,15 +76,10 @@ pipeline {
             for (cs in currentBuild.changeSets) {
               for (entry in cs.items) changed.addAll(entry.affectedPaths)
             }
-            // Only client-facing changes rebuild (and so republish) a
-            // platform; Markdown (.md, .markdown) doesn't count. The job's Git path filter
-            // uses the same paths, so other pushes don't start a run at all.
-            // A CI-only change (Jenkinsfile, ci/) takes a manual run.
-            def touches = { prefixes ->
-              changed.any { p -> !(p.toLowerCase() ==~ /.*\.(md|markdown)/) && prefixes.any { p.startsWith(it) } }
-            }
-            buildPc = touches(['DorkNet.ClientMod/', 'tools/build_pc.py'])
-            buildQuest = touches(['DorkNet.Quest/'])
+            def touches = { prefixes -> changed.any { p -> prefixes.any { p.startsWith(it) } } }
+            def common = ['Jenkinsfile', 'ci/jenkins/']
+            buildPc = touches(common + ['DorkNet.ClientMod/', 'tools/build_pc.py'])
+            buildQuest = touches(common + ['DorkNet.Quest/'])
             echo "Push build; ${changed.size()} changed file(s)"
           } else {
             buildPc = params.PLATFORMS in ['both', 'pc']
@@ -226,15 +220,10 @@ pipeline {
       archiveArtifacts artifacts: 'out/downloads/**', allowEmptyArchive: true, fingerprint: true
     }
     cleanup {
-      // Runs after archiving. Drops the build output and scratch (several
-      // GB) but keeps the Git checkout: the job's path filter compares
-      // pushes against it, and without a workspace Jenkins treats every
-      // push as a change and starts a run. The restic cache and restored
-      // stock builds live outside the workspace ($DORKNET_CACHE).
-      cleanWs(deleteDirs: true, notFailBuild: true, patterns: [
-        [pattern: 'out/**', type: 'INCLUDE'],
-        [pattern: 'quest-work/**', type: 'INCLUDE'],
-      ])
+      // Runs after archiving. Keeps the restic cache and restored stock
+      // builds ($DORKNET_CACHE on the agent, outside the workspace); drops
+      // the workspace's build output and scratch.
+      cleanWs(deleteDirs: true, notFailBuild: true)
     }
   }
 }

ci/jenkins/README.md

@@ -60,21 +60,10 @@ Resolve a table for another build locally with `build.py build --dump-cs`.
 
 ## Triggers
 
-- **Push** to the branch (GitHub webhook) builds only for client-facing
-  changes, and only the platform they touch:
-  - `DorkNet.ClientMod/` and `tools/build_pc.py` rebuild PC;
+- **Push** to the branch (GitHub webhook) rebuilds only what changed:
+  - `DorkNet.ClientMod/` and the PC scripts rebuild PC;
   - `DorkNet.Quest/` rebuilds Quest;
-  - Markdown files (`.md`, `.markdown`) never count; server, docs and CI-only pushes (including
-    the `Jenkinsfile` and `ci/`) build nothing. Run those manually.
-
-  Set the same filter on the job so other pushes don't start a run at all:
-  under the job's **Git** source, add **Additional Behaviours → Polling
-  ignores commits in certain paths** with included regions
-  `DorkNet[.]ClientMod/.*`, `DorkNet[.]Quest/.*`, `tools/build_pc[.]py` (one
-  per line) and excluded region `.*[.](md|markdown)`. The filter compares against the
-  last build's checkout, so the pipeline keeps it and deletes only `out/`
-  and its scratch folders after each run; without a checkout, Jenkins treats
-  every push as a change.
+  - `Jenkinsfile` and `ci/jenkins/` rebuild both.
 - **Build with Parameters** runs a manual build:
   - `PLATFORMS` (both / pc / quest), and `PC_FULL_CLIENT` for the full PC client;
   - snapshot tags, and per-run repository overrides;