HYPERFLEET-618: add PostgreSQL advisory locks for migration coordination

cmd/hyperfleet-api/migrate/cmd.go

@@ -53,11 +53,11 @@ func runMigrateWithError() error {
 		}
 	}()
 
-	if err := db.Migrate(connection.New(ctx)); err != nil {
+	// Use MigrateWithLock to prevent concurrent migrations from multiple pods
+	if err := db.MigrateWithLock(ctx, connection); err != nil {
 		logger.WithError(ctx, err).Error("Migration failed")
 		return err
 	}
 
-	logger.Info(ctx, "Migration completed successfully")
 	return nil
 }

docs/database.md

@@ -61,6 +61,48 @@ Uses GORM AutoMigrate:
 - Additive (creates missing tables, columns, indexes)
 - Run via `./bin/hyperfleet-api migrate`
 
+### Migration Coordination
+
+**Problem:** During rolling deployments, multiple pods attempt to run migrations simultaneously, causing race conditions and deployment failures.
+
+**Solution:** PostgreSQL advisory locks ensure exclusive migration execution.
+
+#### How It Works
+
+```go
+// Only one pod/process acquires the lock and runs migrations
+// Others wait until the lock is released
+db.MigrateWithLock(ctx, factory)
+```
+
+**Implementation:**
+1. Pod sets statement timeout (5 minutes) to prevent indefinite blocking
+2. Pod acquires advisory lock via `pg_advisory_xact_lock(hash("migrations"), hash("Migrations"))`
+3. Lock holder runs migrations exclusively
+4. Other pods block until lock is released or timeout is reached
+5. Lock automatically released on transaction commit
+
+**Key Features:**
+- **Zero infrastructure overhead** - Uses native PostgreSQL locks
+- **Automatic cleanup** - Locks released on transaction end or pod crash
+- **Timeout protection** - 5-minute timeout prevents indefinite blocking if a pod hangs
+- **Nested lock support** - Same lock can be acquired in nested contexts without deadlock
+- **UUID-based ownership** - Only original acquirer can unlock
+
+#### Testing Concurrent Migrations
+
+Integration tests validate concurrent behavior:
+
+```bash
+make test-integration  # Runs TestConcurrentMigrations
+```
+
+**Test coverage:**
+- `TestConcurrentMigrations` - Multiple pods running migrations simultaneously
+- `TestAdvisoryLocksConcurrently` - Lock serialization under race conditions
+- `TestAdvisoryLocksWithTransactions` - Lock + transaction interaction
+- `TestAdvisoryLockBlocking` - Lock blocking behavior
+
 ## Database Setup
 
 ```bash

pkg/config/db.go

@@ -12,10 +12,11 @@ import (
 )
 
 type DatabaseConfig struct {
-	Dialect            string `json:"dialect"`
-	SSLMode            string `json:"sslmode"`
-	Debug              bool   `json:"debug"`
-	MaxOpenConnections int    `json:"max_connections"`
+	Dialect                    string `json:"dialect"`
+	SSLMode                    string `json:"sslmode"`
+	Debug                      bool   `json:"debug"`
+	MaxOpenConnections         int    `json:"max_connections"`
+	AdvisoryLockTimeoutSeconds int    `json:"advisory_lock_timeout_seconds"`
 
 	ConnMaxLifetime    time.Duration `json:"conn_max_lifetime"`
 	ConnMaxIdleTime    time.Duration `json:"conn_max_idle_time"`
@@ -40,10 +41,11 @@ type DatabaseConfig struct {
 
 func NewDatabaseConfig() *DatabaseConfig {
 	return &DatabaseConfig{
-		Dialect:            "postgres",
-		SSLMode:            "disable",
-		Debug:              false,
-		MaxOpenConnections: 50,
+		Dialect:                    "postgres",
+		SSLMode:                    "disable",
+		Debug:                      false,
+		MaxOpenConnections:         50,
+		AdvisoryLockTimeoutSeconds: 300, // 5 minutes - prevents indefinite blocking on migrations
 
 		ConnMaxLifetime:    5 * time.Minute,
 		ConnMaxIdleTime:    1 * time.Minute,
@@ -74,6 +76,11 @@ func (c *DatabaseConfig) AddFlags(fs *pflag.FlagSet) {
 		&c.MaxOpenConnections, "db-max-open-connections", c.MaxOpenConnections,
 		"Maximum open DB connections for this instance",
 	)
+
+	fs.IntVar(
+		&c.AdvisoryLockTimeoutSeconds, "db-advisory-lock-timeout", c.AdvisoryLockTimeoutSeconds,
+		"Advisory lock timeout in seconds (prevents indefinite blocking during migrations)",
+	)
 	fs.DurationVar(&c.ConnMaxLifetime, "db-conn-max-lifetime", c.ConnMaxLifetime, "Maximum lifetime of a DB connection")
 	fs.DurationVar(&c.ConnMaxIdleTime, "db-conn-max-idle-time", c.ConnMaxIdleTime, "Maximum idle time of a DB connection")
 	fs.IntVar(&c.MaxIdleConnections, "db-max-idle-connections", c.MaxIdleConnections, "Maximum idle DB connections")
@@ -96,6 +103,15 @@ func (c *DatabaseConfig) BindEnv(fs *pflag.FlagSet) {
 			}
 		}
 	}
+
+	if val := os.Getenv("DB_ADVISORY_LOCK_TIMEOUT"); val != "" {
+		if fs == nil || !fs.Changed("db-advisory-lock-timeout") {
+			timeout, err := strconv.Atoi(val)
+			if err == nil && timeout > 0 {
+				c.AdvisoryLockTimeoutSeconds = timeout
+			}
+		}
+	}
 }
 
 func (c *DatabaseConfig) ReadFiles() error {