From ef6b8851cfe85e7f3d880ea41b3234cc903ad030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E4=B8=80=E4=B9=8B?= Date: Thu, 13 Aug 2026 00:48:16 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20database:=20=E5=B0=86?= =?UTF-8?q?=20PostgreSQL=20=E9=87=8D=E6=9E=84=E4=B8=BA=20MySQL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 2 +- README.md | 2 +- cmd/server/main.go | 1 - cmd/synce2e/main.go | 2 +- cmd/synce2e/seed.go | 55 +++++++---- configs/config.example.yaml | 4 +- deploy/README.md | 15 ++- deploy/config.docker.yaml | 4 +- deploy/docker-compose.yml | 18 ++-- docs/architecture.md | 10 +- docs/develop.md | 2 +- .../verification-report-template.md | 2 +- docs/specs/2026-08-07-gitea-k3s-deploy.md | 4 +- .../specs/2026-08-07-multi-instance-safety.md | 14 +-- .../2026-08-08-drop-device-capabilities.md | 8 +- docs/testing.md | 10 +- docs/verification.md | 2 +- e2e/README.md | 4 +- e2e/package.json | 1 + e2e/playwright.runner.config.ts | 7 ++ e2e/run-e2e-web.mjs | 29 ++++-- e2e/web/runner-config.spec.ts | 19 +++- e2e/webe2e/main.go | 56 +++++++---- go.mod | 9 +- go.sum | 10 -- .../device_ctr/register_web_test.go | 2 +- .../device_token_entity/device_token.go | 2 +- internal/model/entity/sync_entity/sync.go | 4 +- .../user_identity_entity/user_identity.go | 2 +- .../device_flow_repo/device_flow_test.go | 12 +-- internal/repository/device_repo/device.go | 22 +++-- .../repository/device_repo/device_test.go | 26 ++--- .../device_token_repo/device_token_test.go | 22 ++--- internal/repository/follow_repo/follow.go | 2 +- .../repository/follow_repo/follow_test.go | 14 +-- internal/repository/sync_repo/avatar.go | 2 +- internal/repository/sync_repo/object.go | 21 ++-- internal/repository/sync_repo/state.go | 16 ++- internal/repository/sync_repo/sync_test.go | 62 ++++++------ .../user_identity_repo/user_identity.go | 3 + .../user_identity_repo/user_identity_test.go | 10 +- internal/repository/user_repo/user_test.go | 10 +- internal/service/device_svc/client_info.go | 2 +- internal/service/device_svc/device.go | 4 +- internal/service/device_svc/device_test.go | 2 +- .../service/device_svc/register_web_test.go | 6 +- internal/service/sync_svc/sync.go | 2 +- internal/service/sync_svc/sync_test.go | 2 +- internal/service/user_svc/user_test.go | 2 +- internal/testutils/database.go | 33 +++++-- migrations/202605200001_users.go | 13 +-- migrations/202605200002_user_identities.go | 19 ++-- migrations/202605200003_devices.go | 19 ++-- migrations/202605200004_device_tokens.go | 16 +-- migrations/202605200005_device_flow_codes.go | 22 ++--- migrations/202608090001_workspace_sync.go | 79 ++++++++------- migrations/202608100001_followed_sessions.go | 14 +-- migrations/migrations.go | 43 ++++---- migrations/migrations_test.go | 98 ------------------- 59 files changed, 440 insertions(+), 458 deletions(-) create mode 100644 e2e/playwright.runner.config.ts delete mode 100644 migrations/migrations_test.go diff --git a/AGENTS.md b/AGENTS.md index 35c39841..99ebbe42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ Agent guidance for `agentre-server`. AgentRe Server — SaaS backend. Accounts + RFC 8628 Device Flow. -Go 1.26 on the [cago](https://github.com/cago-frame/cago) framework, PostgreSQL 18 + Redis 7, +Go 1.26 on the [cago](https://github.com/cago-frame/cago) framework, MySQL 9.7 + Redis 7, with a React 19 + Vite + Tailwind + shadcn frontend embedded into the binary via `//go:embed`. Module path is the bare `agentre-server` (not a GitHub path — deliberate, it is not imported by anything). diff --git a/README.md b/README.md index 60736c1e..ab28a5df 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ curl http://localhost:8443/v1/healthz ```bash cp configs/config.example.yaml configs/config.yaml # gitignored runtime 配置 -# 把 db.dsn / redis.addr 指向你自己的 PostgreSQL + Redis +# 把 db.dsn / redis.addr 指向你自己的 MySQL + Redis make dev ``` diff --git a/cmd/server/main.go b/cmd/server/main.go index 99b2587a..3b17d12b 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -7,7 +7,6 @@ import ( "github.com/cago-frame/cago" "github.com/cago-frame/cago/configs" "github.com/cago-frame/cago/database/db" - _ "github.com/cago-frame/cago/database/db/postgres" "github.com/cago-frame/cago/pkg/component" "github.com/cago-frame/cago/pkg/opentelemetry/metric" "github.com/cago-frame/cago/pkg/opentelemetry/trace" diff --git a/cmd/synce2e/main.go b/cmd/synce2e/main.go index d8b87f07..e33260c3 100644 --- a/cmd/synce2e/main.go +++ b/cmd/synce2e/main.go @@ -7,7 +7,7 @@ // - **seed / cleanup** — the desktop logs in through RFC 8628 Device Flow whose // terminus is GitHub OAuth, which nobody can click in an e2e. `seed` writes an // account + its devices + one refresh token per device straight into -// PostgreSQL; every run gets its own account and its own fingerprints, and +// MySQL; every run gets its own account and its own fingerprints, and // `cleanup` removes exactly the rows that run created (scoped by user id — it // never truncates and never touches a row it did not write). // - **peer** — a simulated second desktop that speaks the same `/v1/sync/*` diff --git a/cmd/synce2e/seed.go b/cmd/synce2e/seed.go index 32ea7151..bbd61c06 100644 --- a/cmd/synce2e/seed.go +++ b/cmd/synce2e/seed.go @@ -12,7 +12,7 @@ import ( "strings" "time" - "gorm.io/driver/postgres" + "gorm.io/driver/mysql" "gorm.io/gorm" gormlogger "gorm.io/gorm/logger" ) @@ -43,24 +43,24 @@ func openDB(dsn string) (*gorm.DB, error) { if strings.TrimSpace(dsn) == "" { return nil, fmt.Errorf("--dsn is required (the harness reads it from agentre-server/configs/config.yaml)") } - gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: gormlogger.Discard}) + gdb, err := gorm.Open(mysql.Open(dsn), &gorm.Config{Logger: gormlogger.Discard}) if err != nil { // A dead database must be loud: the suite may not silently skip or pretend to pass. - return nil, fmt.Errorf("connect postgres: %w", err) + return nil, fmt.Errorf("connect mysql: %w", err) } sqlDB, err := gdb.DB() if err != nil { return nil, err } if err := sqlDB.Ping(); err != nil { - return nil, fmt.Errorf("ping postgres: %w", err) + return nil, fmt.Errorf("ping mysql: %w", err) } return gdb, nil } func runSeed(args []string) error { fs := flag.NewFlagSet("seed", flag.ExitOnError) - dsn := fs.String("dsn", os.Getenv("SYNCE2E_DSN"), "PostgreSQL DSN") + dsn := fs.String("dsn", os.Getenv("SYNCE2E_DSN"), "MySQL DSN") runID := fs.String("run-id", "", "unique id for this run") devices := fs.String("devices", "", "comma separated name:kind:platform triples") if err := fs.Parse(args); err != nil { @@ -81,13 +81,20 @@ func runSeed(args []string) error { now := time.Now().UnixMilli() out := &seedResult{RunID: *runID, Email: accountEmail(*runID)} err = gdb.Transaction(func(tx *gorm.DB) error { - if err := tx.Raw( - `INSERT INTO users (email, email_verified, display_name, avatar_url, status, createtime, updatetime) - VALUES (?, true, ?, '', 1, ?, ?) RETURNING id`, - out.Email, "synce2e "+*runID, now, now, - ).Scan(&out.UserID).Error; err != nil { + user := struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement"` + Email string `gorm:"column:email"` + EmailVerified bool `gorm:"column:email_verified"` + DisplayName string `gorm:"column:display_name"` + AvatarURL string `gorm:"column:avatar_url"` + Status int `gorm:"column:status"` + Createtime int64 `gorm:"column:createtime"` + Updatetime int64 `gorm:"column:updatetime"` + }{Email: out.Email, EmailVerified: true, DisplayName: "synce2e " + *runID, Status: 1, Createtime: now, Updatetime: now} + if err := tx.Table("users").Create(&user).Error; err != nil { return fmt.Errorf("insert user: %w", err) } + out.UserID = user.ID for _, spec := range specs { dev := &seededDevice{ Name: spec.name, @@ -95,13 +102,27 @@ func runSeed(args []string) error { Platform: spec.platform, Fingerprint: fmt.Sprintf("synce2e-%s-%s", *runID, spec.name), } - if err := tx.Raw( - `INSERT INTO devices (user_id, name, kind, platform, version, fingerprint, last_seen_at, status, createtime, updatetime) - VALUES (?, ?, ?, ?, 'e2e', ?, ?, 1, ?, ?) RETURNING id`, - out.UserID, dev.Name, dev.Kind, dev.Platform, dev.Fingerprint, now, now, now, - ).Scan(&dev.DeviceID).Error; err != nil { + row := struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement"` + UserID int64 `gorm:"column:user_id"` + Name string `gorm:"column:name"` + Kind string `gorm:"column:kind"` + Platform string `gorm:"column:platform"` + Version string `gorm:"column:version"` + Fingerprint string `gorm:"column:fingerprint"` + LastSeenAt int64 `gorm:"column:last_seen_at"` + Status int `gorm:"column:status"` + Createtime int64 `gorm:"column:createtime"` + Updatetime int64 `gorm:"column:updatetime"` + }{ + UserID: out.UserID, Name: dev.Name, Kind: dev.Kind, Platform: dev.Platform, + Version: "e2e", Fingerprint: dev.Fingerprint, LastSeenAt: now, + Status: 1, Createtime: now, Updatetime: now, + } + if err := tx.Table("devices").Create(&row).Error; err != nil { return fmt.Errorf("insert device %s: %w", spec.name, err) } + dev.DeviceID = row.ID plain, err := randomToken() if err != nil { return err @@ -169,7 +190,7 @@ type cleanupResult struct { func runCleanup(args []string) error { fs := flag.NewFlagSet("cleanup", flag.ExitOnError) - dsn := fs.String("dsn", os.Getenv("SYNCE2E_DSN"), "PostgreSQL DSN") + dsn := fs.String("dsn", os.Getenv("SYNCE2E_DSN"), "MySQL DSN") runID := fs.String("run-id", "", "run id used at seed time") if err := fs.Parse(args); err != nil { return err @@ -253,7 +274,7 @@ type localPathRow struct { // device_local_paths namespace actually holds, per device. func runLocalPaths(args []string) error { fs := flag.NewFlagSet("local-paths", flag.ExitOnError) - dsn := fs.String("dsn", os.Getenv("SYNCE2E_DSN"), "PostgreSQL DSN") + dsn := fs.String("dsn", os.Getenv("SYNCE2E_DSN"), "MySQL DSN") runID := fs.String("run-id", "", "run id used at seed time") if err := fs.Parse(args); err != nil { return err diff --git a/configs/config.example.yaml b/configs/config.example.yaml index 387383dd..f67c7221 100644 --- a/configs/config.example.yaml +++ b/configs/config.example.yaml @@ -7,8 +7,8 @@ http: - "0.0.0.0:8443" db: - driver: postgres - dsn: "postgres://server:server@127.0.0.1:5432/server?sslmode=disable" + driver: mysql + dsn: "server:server@tcp(127.0.0.1:3306)/server?charset=utf8mb4&parseTime=True&loc=Local" debug: false prepareStmt: false diff --git a/deploy/README.md b/deploy/README.md index 00b9dbd5..05dd3bc3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -3,7 +3,7 @@ ``` deploy/ Dockerfile 镜像：前端和后端都在里面构建，产物是单个静态二进制 - docker-compose.yml 单机部署：server + PostgreSQL + Redis + docker-compose.yml 单机部署：server + MySQL + Redis config.docker.yaml compose 用的配置 helm/ Kubernetes 部署 ``` @@ -58,12 +58,11 @@ curl http://localhost:8443/v1/healthz `{"db_ping":true,"redis":true}` 就是好了，浏览器打开 能看到界面。 -数据落在仓库根的 `data/pg` 和 `data/redis`，删掉就等于重置。 +数据落在仓库根的 `data/mysql` 和 `data/redis`，删掉就等于重置。 -Compose 固定使用 PostgreSQL 18.4。PostgreSQL 不能跨大版本直接读取旧数据目录；如果 -`data/pg` 来自 PostgreSQL 16 或 17，先用旧版本导出，再导入全新的 PostgreSQL 18 -数据目录（或者按 PostgreSQL 官方流程运行 `pg_upgrade`），不要直接执行 `up -d` -让 18 读取旧目录。 +Compose 固定使用 MySQL 9.7.2。升级 MySQL 前先做逻辑备份，并按 MySQL 官方 +升级路径检查目标版本是否支持直接读取当前数据目录；不要让不兼容的大版本 +直接复用 `data/mysql`。 ### 要改配置 @@ -116,7 +115,7 @@ docker run --rm -p 8443:8443 \ ## Kubernetes 部署 -`helm/` 下是 chart，只部署服务本身——PostgreSQL、Redis、etcd 都用集群里现成的。 +`helm/` 下是 chart，只部署服务本身——MySQL、Redis、etcd 都用集群里现成的。 ```bash helm upgrade --install agentre-server ./deploy/helm \ @@ -151,7 +150,7 @@ k8s 上只有四个引导键从 ConfigMap 进容器（`env`、`debug`、`source` | key | 内容 | | --- | --- | | `logger` | 日志级别。**`logFile.enable` 必须是 `false`**，容器是只读根文件系统，写不了文件 | -| `db` | PostgreSQL 连接串 | +| `db` | MySQL 连接串 | | `redis` | Redis 地址 | | `http` | 监听地址，端口要和 chart 的 `containerPort` 一致 | | `server` | 域名、会话、JWT 密钥、GitHub OAuth。密钥类的都在这里面 | diff --git a/deploy/config.docker.yaml b/deploy/config.docker.yaml index 896c89d2..25757318 100644 --- a/deploy/config.docker.yaml +++ b/deploy/config.docker.yaml @@ -8,8 +8,8 @@ http: - "0.0.0.0:8443" db: - driver: postgres - dsn: "postgres://server:server@pg:5432/server?sslmode=disable" + driver: mysql + dsn: "server:server@tcp(mysql:3306)/server?charset=utf8mb4&parseTime=True&loc=Local" debug: false prepareStmt: false diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index f2ec169d..21eade42 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -18,19 +18,19 @@ services: AGENTRE_SERVER_OAUTH_GITHUB_CLIENT_SECRET: "${GH_CLIENT_SECRET:-}" AGENTRE_SERVER_SESSION_SECRET: "${SESSION_SECRET:-}" depends_on: - pg: { condition: service_healthy } + mysql: { condition: service_healthy } redis: { condition: service_started } - pg: - image: postgres:18.4-alpine + mysql: + image: mysql:9.7.2 environment: - POSTGRES_USER: server - POSTGRES_PASSWORD: server - POSTGRES_DB: server - # PostgreSQL 18 按大版本保存数据，挂父目录才能支持后续 pg_upgrade。 - volumes: ["../data/pg:/var/lib/postgresql"] + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: server + MYSQL_USER: server + MYSQL_PASSWORD: server + volumes: ["../data/mysql:/var/lib/mysql"] healthcheck: - test: ["CMD", "pg_isready", "-U", "server"] + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -proot --silent"] interval: 5s retries: 10 diff --git a/docs/architecture.md b/docs/architecture.md index 5b1bc9d1..e0519b49 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -171,7 +171,7 @@ The deployment runs multiple replicas by default: `deploy/helm/values.yaml` sets `autoscaling.enabled: true` with `minReplicas: 2`. "There is only one of me" is never a safe assumption — it is false from the first install, not just under load. -**Shared vs process-local state.** PostgreSQL and Redis are the only state visible to the +**Shared vs process-local state.** MySQL and Redis are the only state visible to the whole fleet. Anything else — a `sync.Mutex`, a package-level cache, cago's in-process `cron.Cron()` schedule — lives in one replica's memory and is invisible to its siblings. If something must happen exactly once, or must see what every replica has done, it has to @@ -194,7 +194,7 @@ the transaction and a "deny" committed in that gap would otherwise still hand th token. A write with no conditional `UPDATE` to hang the decision on needs the database to arbitrate -some other way. `device_repo.Upsert` is a single `INSERT … ON CONFLICT ("user_id", +some other way. `device_repo.Upsert` is a single `INSERT … ON DUPLICATE KEY ("user_id", "fingerprint") DO UPDATE`, not a find-then-create, so two exchanges for the same device converge on one row instead of racing to a `uk_devices_user_fingerprint` duplicate-key 500. @@ -215,11 +215,11 @@ replica's lock — TryLock-and-let-expire avoids that. A replica that loses the **Startup one-shot work.** Work that must run exactly once across a concurrently-starting fleet — migrations are the current example — needs a distributed lock, not just an -in-process guard. `migrations/migrations.go`'s `RunMigrations` takes a PostgreSQL advisory +in-process guard. `migrations/migrations.go`'s `RunMigrations` takes a MySQL named lock (`withMigrationLock`) on a connection obtained via `sqlDB.Conn(ctx)` before running -gormigrate, because advisory locks are session-scoped and a `*gorm.DB` call can otherwise +gormigrate, because named locks are session-scoped and a `*gorm.DB` call can otherwise land on a different pooled connection than the one that acquired the lock. It polls -`pg_try_advisory_lock` rather than blocking, up to a 120s budget, so a replica that can't +`GET_LOCK` rather than blocking, up to a 120s budget, so a replica that can't get the lock fails loudly instead of hanging past its startup probe. ## How to add an X diff --git a/docs/develop.md b/docs/develop.md index 8b6de7ad..ef7bfd29 100644 --- a/docs/develop.md +++ b/docs/develop.md @@ -38,7 +38,7 @@ The repo has **no build tags at all**, so `make test` runs everything there is. ```bash cp configs/config.example.yaml configs/config.yaml # gitignored runtime config cp .env.example .env # secrets -# point db.dsn / redis.addr in configs/config.yaml at your own PostgreSQL + Redis +# point db.dsn / redis.addr in configs/config.yaml at your own MySQL + Redis make dev ``` diff --git a/docs/references/verification-report-template.md b/docs/references/verification-report-template.md index 36a7a723..89cc51e3 100644 --- a/docs/references/verification-report-template.md +++ b/docs/references/verification-report-template.md @@ -18,7 +18,7 @@ One or two sentences. The claim being tested — not a changelog. ## How I verified it -Environment (mocked / real PostgreSQL + Redis / against staging), and the commands run. +Environment (mocked / real MySQL + Redis / against staging), and the commands run. ```bash go run ./cmd/server diff --git a/docs/specs/2026-08-07-gitea-k3s-deploy.md b/docs/specs/2026-08-07-gitea-k3s-deploy.md index 5fac6273..cc330a5c 100644 --- a/docs/specs/2026-08-07-gitea-k3s-deploy.md +++ b/docs/specs/2026-08-07-gitea-k3s-deploy.md @@ -45,7 +45,7 @@ | # | Decision | Basis and rejected option | |---|---|---| | 1 | 域名 `app.agentrehub.com`，非生产环境加前缀（`test.` / `pre.`） | 用户决定。Rejected: `server.agentre.dev`（`configs/config.example.yaml:47` 的旧值）——用户已改用新域名 | -| 2 | chart 只部署 server 自身，PostgreSQL / Redis 复用集群已有实例 | 用户决定。Rejected: 用 bitnami subchart 一起部署——`helm uninstall` 会连数据一起带走，生产库不该受应用 chart 的生命周期管辖 | +| 2 | chart 只部署 server 自身，MySQL / Redis 复用集群已有实例 | 用户决定。Rejected: 用 bitnami subchart 一起部署——`helm uninstall` 会连数据一起带走，生产库不该受应用 chart 的生命周期管辖 | | 3 | 配置走 etcd 配置中心，ConfigMap 只放引导配置 | 用户决定，与 scriptlist 同源。Rejected: ConfigMap + Secret 全量下发——改配置要重新 `helm upgrade` | | 4 | 不新增 Go 代码即可启用 etcd 配置源 | `pkg/component/core.go:8` 已经 blank-import 了 `configs/etcd`，而 `cmd/server/main.go` 导入了 `pkg/component`，`init()` 已把 `sources["etcd"]` 注册好。Rejected: 在 main.go 里再加一次 blank import——重复且无效果 | | 5 | ConfigMap 必须同时含 `env`、`debug`、`source`、`etcd` 四个键 | 文件源在键缺失时会**回写配置文件**（`configs/file/file.go:33-43`），而 subPath 挂载的 ConfigMap 是只读的，缺一个键就 CrashLoop。这四个键正好是切换到 etcd 之前会被读到的全部键 | @@ -104,7 +104,7 @@ Service 以 80 端口对内暴露，转发到容器的 8443。Ingress 用 `k3s-m ## Out of scope - 让 `/v1/healthz` 在 DB/Redis 不通时返回非 200：那是改 controller 的可观测行为，需要单独的 spec 与回归测试。 -- 部署 PostgreSQL / Redis / etcd 本身（决策 2、3）。 +- 部署 MySQL / Redis / etcd 本身（决策 2、3）。 - 证书签发（cert-manager 等）：chart 只引用已存在的 TLS Secret。 - 改动 `.github/workflows/ci.yml` 与任何 Go 生产代码。GitHub 侧的门禁保持原样，本轮不因为 Gitea 的存在去动它。 （交付后经用户要求追加：`Dockerfile` 与 `docker-compose.yml` 移入 `deploy/`，`docs/deploy.md` 改写为面向人的 diff --git a/docs/specs/2026-08-07-multi-instance-safety.md b/docs/specs/2026-08-07-multi-instance-safety.md index 8d1a987d..1d632ecd 100644 --- a/docs/specs/2026-08-07-multi-instance-safety.md +++ b/docs/specs/2026-08-07-multi-instance-safety.md @@ -6,7 +6,7 @@ **Objective:** 让 agentre-server 在多副本同时运行时行为正确——启动期不会因并发迁移而起不来，定时任务不会每副本各跑一遍，请求路径上不再有依赖「进程内只有我一个」的假设；并把这条约束写进文档，让后续改动有据可依。 -**Hard invariant:** 仓库仍然零 build tag；`make lint` / `make test` 仍是唯一门禁且不新增外部依赖（不引入 Docker、不连真 PG/Redis 跑单元测试）；依赖方向 `controller → service → repository → entity` 不变，`internal/pkg/*` 仍不导入 service/repository；不改动任何既有迁移。 +**Hard invariant:** 仓库仍然零 build tag；`make lint` / `make test` 仍是唯一门禁且不新增外部依赖（不引入 Docker、不连真 MySQL/Redis 跑单元测试）；依赖方向 `controller → service → repository → entity` 不变，`internal/pkg/*` 仍不导入 service/repository；不改动任何既有迁移。 ## Problem @@ -43,10 +43,10 @@ | # | Decision | Basis and rejected option | |---|---|---| -| 1 | 8 项修复放同一轮交付 | 用户决定。Rejected: 拆成「基础设施加锁」与「Device Flow 状态机原子化」两轮——后者会改变并发竞败方的可观察错误、按 `docs/testing.md` 需走真 PG 的 scratch 验证，风险类别与前者不同，分开评审和回滚的粒度更细。用户已知该取舍并选择一轮完成 | +| 1 | 8 项修复放同一轮交付 | 用户决定。Rejected: 拆成「基础设施加锁」与「Device Flow 状态机原子化」两轮——后者会改变并发竞败方的可观察错误、按 `docs/testing.md` 需走真 MySQL 的 scratch 验证，风险类别与前者不同，分开评审和回滚的粒度更细。用户已知该取舍并选择一轮完成 | | 2 | 多实例约束只写文档，不加机械守卫 | 用户决定。 | -| 3 | 迁移用 PostgreSQL advisory lock 串行化，锁持有在一条专用连接上 | advisory lock 是**会话级**的，而 gorm 从连接池取连接，直接 `gdb.Exec("pg_advisory_lock")` 可能在 A 连接上加锁、在 B 连接上跑迁移，锁会随 A 归还池中而失去意义。因此从 `sqlDB.Conn(ctx)` 取一条固定连接持锁，迁移本身照常走连接池——advisory lock 不绑定数据，这样是正确的。进程崩溃时连接断开，PG 自动释放。Rejected: helm `pre-upgrade` Job 单独跑迁移——能解决问题，但 `helm upgrade` 之外的启动路径（本地 `make dev`、`docker-compose`、手工 `kubectl run`）就不再有保护，问题从代码里搬到了部署方式里 | -| 4 | 用 `pg_try_advisory_lock` 轮询而非阻塞的 `pg_advisory_lock` | 阻塞版没有上界，前面的副本卡住会让后面的副本静默挂起到被探针杀掉，日志里什么都看不到。轮询版有明确的等待预算，超时就返回错误、由 `main.go` 打日志退出，CrashLoop 是可见且自愈的。副作用是可以用 sqlmock 断言「拿不到锁时会重试、拿到后才跑迁移、结束后解锁」这个序列 | +| 3 | 迁移用 MySQL named lock 串行化，锁持有在一条专用连接上 | advisory lock 是**会话级**的，而 gorm 从连接池取连接，直接 `gdb.Exec("GET_LOCK")` 可能在 A 连接上加锁、在 B 连接上跑迁移，锁会随 A 归还池中而失去意义。因此从 `sqlDB.Conn(ctx)` 取一条固定连接持锁，迁移本身照常走连接池——advisory lock 不绑定数据，这样是正确的。进程崩溃时连接断开，MySQL 自动释放。Rejected: helm `pre-upgrade` Job 单独跑迁移——能解决问题，但 `helm upgrade` 之外的启动路径（本地 `make dev`、`docker-compose`、手工 `kubectl run`）就不再有保护，问题从代码里搬到了部署方式里 | +| 4 | 用 `GET_LOCK` 轮询而非阻塞的 `GET_LOCK` | 阻塞版没有上界，前面的副本卡住会让后面的副本静默挂起到被探针杀掉，日志里什么都看不到。轮询版有明确的等待预算，超时就返回错误、由 `main.go` 打日志退出，CrashLoop 是可见且自愈的。副作用是可以用 sqlmock 断言「拿不到锁时会重试、拿到后才跑迁移、结束后解锁」这个序列 | | 5 | 迁移等待预算 120s，同时把 chart 的 `startupProbe.failureThreshold` 从 30 提到 60 | 现值 `periodSeconds: 5 × failureThreshold: 30` = 150s 总预算，而等待锁 120s 之后还要真正跑迁移，很容易在迁移中途被探针杀掉；`UseTransaction: false` 下被杀在中途会留下半应用的迁移。提到 60（300s）让「等锁 + 迁移」有富余。Rejected: 缩短等待预算——迁移本身可能就要几十秒，等待预算小于它没有意义 | | 6 | 定时任务用 Redis 锁「占用当期」，**不主动解锁**，靠 TTL 自然过期，TTL 取略小于 cron 周期 | cago 的 `pkg/sync` locker 的 `UnlockKey` 是无条件 `DEL`、不校验持有者（`sync/redis.go`），任务一旦跑超 TTL，别的副本已经拿到锁，而先前那个副本的 Unlock 会把**别人的**锁删掉。改成「只 TryLock、不 Unlock」就完全绕开了这个问题：锁的语义正好是「本周期已被某个副本认领」，也正是 cron 需要的语义。TTL 取 `*/5` → 4m、`0 * * * *` → 50m。Rejected: 自己写带 owner token + Lua CAS 删除的锁——正确但等于在仓库里放第二套锁实现，而 cron 场景不需要提前释放；Rejected: 直接用 cago 的 Lock/Unlock 配对——即上述删错锁的缺陷 | | 7 | 拿不到锁时任务返回 `nil` 而非错误 | 「另一个副本正在跑」是预期内的正常路径，不是故障。返回错误会让 cago 的 crontab 包装器（`crontab.go:37`）在每个没抢到锁的副本上打一条 `cron error`，N-1 份噪音会把真正的失败淹掉 | @@ -60,7 +60,7 @@ ## 交付后的可观察行为 -**启动。** 副本并发启动时，只有一个在跑迁移，其余的在 `pg_try_advisory_lock` 上轮询等待，拿到锁后发现无事可做，正常继续启动。等待超过 120s 的副本以非零码退出并在日志里说明是等迁移锁超时；k8s 重启它，下一次通常就能拿到锁。任何一个副本在持锁期间崩溃，连接断开，PG 立即释放锁，不需要人工介入。 +**启动。** 副本并发启动时，只有一个在跑迁移，其余的在 `GET_LOCK` 上轮询等待，拿到锁后发现无事可做，正常继续启动。等待超过 120s 的副本以非零码退出并在日志里说明是等迁移锁超时；k8s 重启它，下一次通常就能拿到锁。任何一个副本在持锁期间崩溃，连接断开，MySQL 立即释放锁，不需要人工介入。 **定时任务。** 每个周期内，两个清理任务在整个副本集里各执行一次。没抢到的副本安静跳过，日志里不产生错误。某个副本在任务中途死掉，锁在 TTL 到期后释放，下一个周期正常继续。 @@ -90,14 +90,14 @@ | 修复项 | 测试位置与手段 | RED 时应看到 | |---|---|---| -| 迁移锁 | `migrations/` 包内，用 `testutils.DatabasePG` 的 sqlmock 断言语句序列 | 现状根本不发出 `pg_try_advisory_lock`，期望落空 | +| 迁移锁 | `migrations/` 包内，用 `testutils.Database` 的 sqlmock 断言语句序列 | 现状根本不发出 `GET_LOCK`，期望落空 | | cron 锁 | `internal/task/` 包内，miniredis（`cago/pkg/utils/testutils.Redis()`）+ 注入的假任务，断言两次「同周期」调用只有一次真正执行 | 现状两次都执行 | | jti | `internal/pkg/jwt/`，N 个 goroutine 并发 `Sign`，断言 jti 全不相同 | `make test-backend` 已经是 `go test -race ./...`，现状直接报 data race | | 限流原子性 | `internal/pkg/ratelimit/`，用 go-redis hook 记录实际下发的命令，断言 `PTTL > 0` 且从未出现独立的 `expire`/`pexpire` 顶层命令 | 现状会记录到独立的 `expire` | | assets 404 | `internal/web/`，httptest 请求 `/assets/does-not-exist.js` | 现状返回 200 + `text/html` | | 三处 RowsAffected | repository 层 sqlmock 断言 WHERE 条件与返回的行数；service 层 mockgen 让 mock 返回 0 行，断言得到对应的 OAuth 错误且事务未提交 | 现状 repository 方法签名里根本没有行数 | -**真并发只能靠人工验证。** sqlmock 与 mockgen 都无法证明两个真实事务竞争时数据库层面的行为，`docs/testing.md` 也明确禁止在单元测试里起真 PostgreSQL。因此迁移锁与三处 TOCTOU 需要按 `docs/verification.md` 的 scratch 流程，连自有 PG 手工验证：并发启动两个进程看迁移是否串行、用同一个 device_code / refresh token 并发打两个请求看是否只有一个成功。证据是数据库里的行数与两个响应体，不是截图。 +**真并发只能靠人工验证。** sqlmock 与 mockgen 都无法证明两个真实事务竞争时数据库层面的行为，`docs/testing.md` 也明确禁止在单元测试里起真 MySQL。因此迁移锁与三处 TOCTOU 需要按 `docs/verification.md` 的 scratch 流程，连自有 MySQL 手工验证：并发启动两个进程看迁移是否串行、用同一个 device_code / refresh token 并发打两个请求看是否只有一个成功。证据是数据库里的行数与两个响应体，不是截图。 ## Links diff --git a/docs/specs/2026-08-08-drop-device-capabilities.md b/docs/specs/2026-08-08-drop-device-capabilities.md index 0b102a5a..4a8dfe43 100644 --- a/docs/specs/2026-08-08-drop-device-capabilities.md +++ b/docs/specs/2026-08-08-drop-device-capabilities.md @@ -18,7 +18,7 @@ 4. **设计稿承诺的能力比后端存在的还多。** `设计稿/agentre-server.pen` 画板 41/44 右栏有一整块「能力卡」（节点 `eAqxN` / `ShTHY` 同列），列出 `compute` / `session.remote_start` / `session.autonomous` / `fs.browse` 四项，并写着「授权时由设备申报，撤销即失效」；画板 43 审计里有「指纹 a91f2c… · 请求 compute」与批注「能力变更只看得到当前值，还没有历史」。后三个键与「能力变更」这件事，后端一行代码都不存在。[`2026-08-07-auth-flow-redesign.md`](2026-08-07-auth-flow-redesign.md) 决策 12 已经因此拒绝把 `session.remote_start` / `fs.browse` / `filesystem.write` 写进词表——那条决策把症状挡在词表外，没有动病根。 -5. **一个概念三处形态，都不影响行为。** 同一份能力 map 在服务端存在三种表示：`devices.capabilities` jsonb 列、JWT 里的字符串数组（`device_entity.Device.CapabilityList()`）、接口上的 `map[string]bool`（另有 `CapabilityMap()`，只有 `device_entity/device_test.go` 在调用它）。三者互相转换的代码是活的，它们表达的约束是死的。 +5. **一个概念三处形态，都不影响行为。** 同一份能力 map 在服务端存在三种表示：`devices.capabilities` JSON 列、JWT 里的字符串数组（`device_entity.Device.CapabilityList()`）、接口上的 `map[string]bool`（另有 `CapabilityMap()`，只有 `device_entity/device_test.go` 在调用它）。三者互相转换的代码是活的，它们表达的约束是死的。 ## Actors and user stories @@ -33,7 +33,7 @@ | 1 | 整体移除能力概念，不保留任何降级形态 | 用户裁决（2026-08-08）：「先不用考虑机器权限，连上了就是完整的权限」。能力今天零处生效（问题 1、5），保留它等于继续对用户做一个不兑现的承诺。Rejected: 保留字段只在界面隐藏——数据仍然进库、进令牌，下一个人还会以为它有意义 | | 2 | 授权确认屏改成一句无条件的完整权限说明，不再有中性 / 风险两档 | 决策 1 之下 `capabilities.compute` 不复存在，而分档依据本来就是自报字段（问题 3）。授权的真实后果与设备类型无关，一句话说完更准确。Rejected: 改按 `device_kind` 分档——正是 `auth-flow-redesign` 决策 7 已经否掉的做法，且四种 kind 拿到的权限完全相同 | | 3 | JWT 的 `caps` claim 一并删除 | 只写不读（问题 1）。留着它会让下一个读令牌的人以为存在能力边界。已签发的旧令牌里多一个字段不影响解析——`jwt.go` 的 `registered` 按字段名反序列化，未知字段被忽略，因此不需要任何兼容分支。Rejected: 保留 claim 但恒为空数组——同样误导，还要为它写一句解释性注释 | -| 4 | 两个 jsonb 列用追加迁移 DROP 掉 | 用户裁决（2026-08-08）。列里只有展示用的自报值，删除不影响登录 / 刷新 / 撤销 / 设备列表任一链路。仓库规矩是追加 patch 迁移、不改既有迁移（AGENTS.md 非协商项 6），Rollback 分支按 `migrations/202608030001_device_tokens_access_jti.go` 的形状补回列。Rejected: 留列只断代码引用——库里留一个没人写也没人读的字段，下次读 schema 的人还得重新考古 | +| 4 | 两个 JSON 列用追加迁移 DROP 掉 | 用户裁决（2026-08-08）。列里只有展示用的自报值，删除不影响登录 / 刷新 / 撤销 / 设备列表任一链路。仓库规矩是追加 patch 迁移、不改既有迁移（AGENTS.md 非协商项 6），Rollback 分支按 `migrations/202608030001_device_tokens_access_jti.go` 的形状补回列。Rejected: 留列只断代码引用——库里留一个没人写也没人读的字段，下次读 schema 的人还得重新考古 | | 5 | 桌面端仓库（`agentre`）同轮清掉生产者 | 用户裁决（2026-08-08）。服务端删字段后 gin 绑定会静默忽略桌面端多发的 `capabilities`，不会出错，但桌面端会一直发一个没人认的字段。两个仓库各自独立提交（工作区规矩：不跨仓混提交）。Rejected: 只动服务端——留一段死代码等下一轮 | | 6 | 设计稿把能力相关表达整体删除，不换成别的信息 | 与决策 1 一致。右栏已有「撤销这台设备」卡承担「这台设备能干什么、怎么停掉它」，再放一块只写一句「拥有完整权限」的卡是纯装饰。Rejected: 把能力卡换成「完整权限」说明卡——增加一块没有操作、也没有可变信息的卡片 | | 7 | 不引入任何形式的设备权限模型作为替代 | 用户裁决（2026-08-08）：「先不用考虑机器权限」。真正的权限模型需要服务端在每条链路上执行判断，是一次独立设计，不是把这套自报字段改个形状留下来。Rejected: 保留一个 `trusted` 布尔位备用——本轮没有消费者，会先于它的用途存在 | @@ -58,9 +58,9 @@ ## 数据 -**R8 — 两列由一条追加到 `migrationList()` 末尾的迁移删除。** `devices.capabilities` 与 `device_flow_codes.client_capabilities` 被 DROP；Rollback 分支按原定义（`jsonb NOT NULL DEFAULT '{}'`）补回两列。这条迁移不触碰任何其他列、索引、约束或数据行，也不修改既有的任何一条迁移。 +**R8 — 两列由一条追加到 `migrationList()` 末尾的迁移删除。** `devices.capabilities` 与 `device_flow_codes.client_capabilities` 被 DROP；Rollback 分支按原定义（`JSON NOT NULL DEFAULT '{}'`）补回两列。这条迁移不触碰任何其他列、索引、约束或数据行，也不修改既有的任何一条迁移。 -**R9 — 设备 upsert 不再写能力列。** 冲突键仍是 `(user_id, fingerprint)`，赋值列去掉 `capabilities`、其余原样，`createtime` 仍不在赋值列里；`RETURNING` 回填不变。 +**R9 — 设备 upsert 不再写能力列。** 冲突键仍是 `(user_id, fingerprint)`，赋值列去掉 `capabilities`、其余原样，`createtime` 仍不在赋值列里；事务内读回最终行。 ## 桌面端（`agentre` 仓库） diff --git a/docs/testing.md b/docs/testing.md index ff75a44c..b5ebde43 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -15,7 +15,7 @@ speculatively. And do not add a guard in the consumer to paper over a producer b | Layer | Tool | Rule | | --- | --- | --- | | Entity | plain table tests | Business rules (`Check`, `IsActive`, state transitions) live here, so test them here | -| Repository | **sqlmock** | **Never start a real PostgreSQL.** `internal/testutils.Database(t)` gives you a postgres-dialect sqlmock through ctx | +| Repository | **sqlmock** | **Never start a real MySQL.** `internal/testutils.Database(t)` gives you a mysql-dialect sqlmock through ctx | | Service | **mockgen** | Inject repo mocks via `xxx_repo.RegisterXxx(mock)`. Never touch a database | | Controller | `muxtest.TestMux` | Build the route tree, `testMux.Do(ctx, req, resp)` | | Migrations | **nothing** for the DDL; sqlmock for the runner around it | `migrationList()` is deliberately untested — see below | @@ -65,7 +65,7 @@ A target you did not run is obvious; a tagged-out test is invisible. ## Migrations are deliberately untested There is no automated check that `migrations/migrationList()` runs cleanly against a real -PostgreSQL — that would mean a Docker dependency, which this suite deliberately avoids. +MySQL — that would mean a Docker dependency, which this suite deliberately avoids. **Know what that costs you.** `cmd/server/main.go` runs `migrations.RunMigrations` at startup, so a migration that is valid Go but invalid SQL — wrong type, bad constraint, @@ -77,7 +77,7 @@ So when you touch `migrations/`, verify it by hand before merging: ```bash make dev # migrations run at startup against db.dsn; watch for errors -psql "" -c '\dt' # then read the tables back directly +mysql --host --user --password -e 'SHOW TABLES' # read tables back directly ``` Write that check up under `e2e/scratch/` per [verification.md](verification.md) — for @@ -85,8 +85,8 @@ migrations the evidence is the table list, not a screenshot. **What is untested is the DDL, not the runner.** `migrations/migrations_test.go` does use sqlmock, on the advisory-lock wrapper `withMigrationLock` that serialises concurrently -starting replicas — it asserts the `pg_try_advisory_lock` retry, that the migration func -only runs once the lock is held, and that `pg_advisory_unlock` follows. That is a +starting replicas — it asserts the `GET_LOCK` retry, that the migration func +only runs once the lock is held, and that `RELEASE_LOCK` follows. That is a statement-sequence assertion, so it stays hermetic; it says nothing about whether any migration in `migrationList()` is valid SQL. diff --git a/docs/verification.md b/docs/verification.md index 936a55d5..46829bcc 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -33,7 +33,7 @@ cd e2e && pnpm scratch spec, use that spec's slug**, so the evidence and the spec are findable from each other. Needing a real backend (real device flow, migrations, session cookies) — the server -takes its PostgreSQL and Redis from `configs/config.yaml`, so point that at your own +takes its MySQL and Redis from `configs/config.yaml`, so point that at your own instances: ```bash diff --git a/e2e/README.md b/e2e/README.md index 6b04145d..c3a8d477 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -86,7 +86,7 @@ separate, deliberate decision, not something that happens because it was handy. ## Needing a real backend The smoke track runs against the vite dev server with the API mocked, so it needs -no PostgreSQL or Redis. Full-stack flows (real device flow, migrations, session +no MySQL or Redis. Full-stack flows (real device flow, migrations, session cookies) need both, plus the Go server: ```bash @@ -102,7 +102,7 @@ the real backend. A spec reaches that backend by simply **not** calling the mocking to switch off. `make dev` works too, but it also starts a second vite on 5174 that the run does not use. -**PostgreSQL and Redis come from `configs/config.yaml`** — the server loads it +**MySQL and Redis come from `configs/config.yaml`** — the server loads it through `configs.NewConfig("agentre-server")` in `cmd/server/main.go`, so `db.dsn` and `redis.addr` decide what a scratch run actually writes to. Point them at your own instances; the file is gitignored and diff --git a/e2e/package.json b/e2e/package.json index 37d42dbf..ee7fc816 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -10,6 +10,7 @@ "scratch": "playwright install chromium && playwright test --config playwright.scratch.config.ts", "web": "node run-e2e-web.mjs", "dual": "node run-e2e-web.mjs --dual", + "runner-test": "playwright test --config playwright.runner.config.ts", "format": "prettier --write .", "lint": "prettier --check ." }, diff --git a/e2e/playwright.runner.config.ts b/e2e/playwright.runner.config.ts new file mode 100644 index 00000000..1421e46d --- /dev/null +++ b/e2e/playwright.runner.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: "web", + testMatch: "runner-config.spec.ts", + reporter: "list", +}); diff --git a/e2e/run-e2e-web.mjs b/e2e/run-e2e-web.mjs index 78142e96..738dc82c 100644 --- a/e2e/run-e2e-web.mjs +++ b/e2e/run-e2e-web.mjs @@ -8,7 +8,7 @@ // // 1. locate the agentre-server and agentre checkouts, read the developer's DSN + Redis // 2. build the server + the `webe2e` harness tool (GOWORK=off) + agentred (-tags e2e) -// 3. start the server on a free port against the developer's PostgreSQL + Redis +// 3. start the server on a free port against the developer's MySQL + Redis // 4. seed ONE throwaway account + one agentred device + one Redis browser session // + the account-level workspace (agents, a project and its path on that // agentred) that the "start a new conversation from the web" flow reads @@ -533,6 +533,15 @@ export function parseDSN(text) { return blockScalar(text, "db", "dsn", "") || null; } +// parseMySQLAddress 只取 Go MySQL DSN 的 tcp 地址，不返回用户名、口令或库名。 +export function parseMySQLAddress(value) { + const match = value?.match(/@tcp\(([^():]+|\[[^\]]+\]):(\d+)\)\//); + if (!match) return null; + const port = Number(match[2]); + if (!Number.isInteger(port) || port < 1 || port > 65535) return null; + return { host: match[1].replace(/^\[|\]$/g, ""), port }; +} + /** parseRedis 读 redis 段。缺省值与 configs/config.example.yaml 一致。 */ export function parseRedis(text) { return { @@ -709,7 +718,7 @@ async function waitForServer(baseURL) { * 没有这样的行时返回 null(由调用方去探测依赖)。 * * 这一步是「说出缺什么」的第一手证据:服务器起不来的原因五花八门(密钥文件读不到、 - * 端口被占、迁移失败……),未经探测就断言「PostgreSQL 或 Redis 不可达」会把读者 + * 端口被占、迁移失败……),未经探测就断言「MySQL 或 Redis 不可达」会把读者 * 引向完全无关的子系统 —— 本轮的运行时验证正是被这句话挡了一次。 */ export function summarizeStartupFailure(logText) { @@ -727,7 +736,7 @@ export function summarizeStartupFailure(logText) { } // startupDiagnosis 组织服务器起不来时给人看的那句话:先服务器日志自己的最后一条 -// 错误(决定性),日志里没有错误行时才去 TCP 探测 PostgreSQL 与 Redis,并如实说 +// 错误(决定性),日志里没有错误行时才去 TCP 探测 MySQL 与 Redis,并如实说 // 哪个连得上、哪个连不上。 async function startupDiagnosis() { let logText = ""; @@ -750,17 +759,17 @@ async function startupDiagnosis() { // probeDependencies TCP 探测配置里那两个依赖,逐个如实报结果。 async function probeDependencies() { const targets = []; - try { - const u = new URL(dsn); + const mysqlAddress = parseMySQLAddress(dsn); + if (mysqlAddress) { targets.push({ - label: `PostgreSQL ${u.hostname}:${u.port || 5432} (db.dsn)`, - host: u.hostname, - port: Number(u.port || 5432), + label: `MySQL ${mysqlAddress.host}:${mysqlAddress.port} (db.dsn)`, + host: mysqlAddress.host, + port: mysqlAddress.port, }); - } catch { + } else { // 不回显 dsn 本身:它带着口令,而这条消息会被贴进日志与报告里。 targets.push({ - label: "PostgreSQL (db.dsn is not a URL this harness can parse)", + label: "MySQL (db.dsn is not a URL this harness can parse)", }); } const [rhost, rport] = redis.addr.split(":"); diff --git a/e2e/web/runner-config.spec.ts b/e2e/web/runner-config.spec.ts index 3fa9bc2d..65b63203 100644 --- a/e2e/web/runner-config.spec.ts +++ b/e2e/web/runner-config.spec.ts @@ -10,7 +10,7 @@ * 在 Keys 非空时**只读列表** —— runner 必须把列表每一项的私钥/公钥路径都改写成 * 绝对路径,只断言第一个同名字段(或只摊平出一个扁平字段)会形成假绿, * server 仍会死在 missing JWT key; - * 3. 服务器起不来时 runner 未经探测就断言「PostgreSQL 或 Redis 不可达」, + * 3. 服务器起不来时 runner 未经探测就断言「MySQL 或 Redis 不可达」, * 把读者引向错误的子系统。 */ import { test, expect } from "@playwright/test"; @@ -21,6 +21,7 @@ import { join } from "node:path"; import { rewriteServerConfig, parseDSN, + parseMySQLAddress, parseRedis, summarizeStartupFailure, } from "../run-e2e-web.mjs"; @@ -156,8 +157,8 @@ test("轮换形态缺了 keys 列表指名的密钥文件时当场失败并说 test("带引号的 DSN 与 Redis 口令读成裸值", () => { const text = `db: - driver: postgres - dsn: "postgres://server:server@127.0.0.1:5432/server?sslmode=disable" + driver: mysql + dsn: "server:server@tcp(127.0.0.1:3306)/server?charset=utf8mb4&parseTime=True&loc=Local" redis: addr: "127.0.0.1:6379" @@ -166,7 +167,7 @@ redis: `; expect(parseDSN(text)).toBe( - "postgres://server:server@127.0.0.1:5432/server?sslmode=disable", + "server:server@tcp(127.0.0.1:3306)/server?charset=utf8mb4&parseTime=True&loc=Local", ); expect(parseRedis(text)).toEqual({ addr: "127.0.0.1:6379", @@ -175,7 +176,15 @@ redis: }); }); -test("启动失败的诊断取服务器日志自己的最后一条错误,不去猜 PG/Redis", () => { +test("MySQL DSN 不暴露口令地取出依赖地址", () => { + expect( + parseMySQLAddress( + "server:secret@tcp(192.168.8.141:3306)/agentre_server_dev?charset=utf8mb4", + ), + ).toEqual({ host: "192.168.8.141", port: 3306 }); +}); + +test("启动失败的诊断取服务器日志自己的最后一条错误,不去猜 MySQL/Redis", () => { const log = `{"level":"info","msg":"config loaded"} {"level":"info","msg":"database connected"} {"level":"fatal","msg":"missing JWT key: open \\"./runtime/keys/jwt.key\\": no such file or directory"} diff --git a/e2e/webe2e/main.go b/e2e/webe2e/main.go index 9efbb2c2..b858953d 100644 --- a/e2e/webe2e/main.go +++ b/e2e/webe2e/main.go @@ -4,7 +4,7 @@ // // The web frontend's login ends at GitHub OAuth, which nobody can click in an // e2e, so the runner seeds the identity straight into the developer's -// PostgreSQL + Redis: +// MySQL + Redis: // // - a throwaway user (the account); // - one kind=agentred device + a refresh token for the real `agentred run` @@ -37,7 +37,7 @@ import ( "time" goredis "github.com/redis/go-redis/v9" - "gorm.io/driver/postgres" + "gorm.io/driver/mysql" "gorm.io/gorm" gormlogger "gorm.io/gorm/logger" ) @@ -144,23 +144,23 @@ func openDB(dsn string) (*gorm.DB, error) { if strings.TrimSpace(dsn) == "" { return nil, fmt.Errorf("--dsn is required (the harness reads it from agentre-server/configs/config.yaml)") } - gdb, err := gorm.Open(postgres.Open(dsn), &gorm.Config{Logger: gormlogger.Discard}) + gdb, err := gorm.Open(mysql.Open(dsn), &gorm.Config{Logger: gormlogger.Discard}) if err != nil { - return nil, fmt.Errorf("connect postgres: %w", err) + return nil, fmt.Errorf("connect mysql: %w", err) } sqlDB, err := gdb.DB() if err != nil { return nil, err } if err := sqlDB.Ping(); err != nil { - return nil, fmt.Errorf("ping postgres: %w", err) + return nil, fmt.Errorf("ping mysql: %w", err) } return gdb, nil } func runSeed(args []string) error { fs := flag.NewFlagSet("seed", flag.ExitOnError) - dsn := fs.String("dsn", os.Getenv("WEBE2E_DSN"), "PostgreSQL DSN") + dsn := fs.String("dsn", os.Getenv("WEBE2E_DSN"), "MySQL DSN") runID := fs.String("run-id", "", "unique id for this run") agentredFP := fs.String("agentred-fingerprint", "", "the agentred device fingerprint (sha256: of the daemon instance uuid)") desktopFP := fs.String("desktop-fingerprint", "", "optional: also seed a kind=desktop device with this fingerprint (the desktop+web dual scenario)") @@ -192,13 +192,20 @@ func runSeed(args []string) error { now := time.Now().UnixMilli() out := &seedResult{RunID: *runID, Email: accountEmail(*runID)} err = gdb.Transaction(func(tx *gorm.DB) error { - if err := tx.Raw( - `INSERT INTO users (email, email_verified, display_name, avatar_url, status, createtime, updatetime) - VALUES (?, true, ?, '', 1, ?, ?) RETURNING id`, - out.Email, "webe2e "+*runID, now, now, - ).Scan(&out.UserID).Error; err != nil { + user := struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement"` + Email string `gorm:"column:email"` + EmailVerified bool `gorm:"column:email_verified"` + DisplayName string `gorm:"column:display_name"` + AvatarURL string `gorm:"column:avatar_url"` + Status int `gorm:"column:status"` + Createtime int64 `gorm:"column:createtime"` + Updatetime int64 `gorm:"column:updatetime"` + }{Email: out.Email, EmailVerified: true, DisplayName: "webe2e " + *runID, Status: 1, Createtime: now, Updatetime: now} + if err := tx.Table("users").Create(&user).Error; err != nil { return fmt.Errorf("insert user: %w", err) } + out.UserID = user.ID agentred, err := seedDevice(tx, out.UserID, agentredDeviceName, "agentred", *agentredFP, now) if err != nil { return err @@ -249,13 +256,26 @@ func runSeed(args []string) error { // 一枚坏令牌会在那一步大声失败，而不是变成半登录的进程。 func seedDevice(tx *gorm.DB, userID int64, name, kind, fingerprint string, now int64) (*seededDevice, error) { out := &seededDevice{Fingerprint: fingerprint} - if err := tx.Raw( - `INSERT INTO devices (user_id, name, kind, platform, version, fingerprint, last_seen_at, status, createtime, updatetime) - VALUES (?, ?, ?, 'darwin', 'e2e', ?, ?, 1, ?, ?) RETURNING id`, - userID, name, kind, fingerprint, now, now, now, - ).Scan(&out.DeviceID).Error; err != nil { + row := struct { + ID int64 `gorm:"column:id;primaryKey;autoIncrement"` + UserID int64 `gorm:"column:user_id"` + Name string `gorm:"column:name"` + Kind string `gorm:"column:kind"` + Platform string `gorm:"column:platform"` + Version string `gorm:"column:version"` + Fingerprint string `gorm:"column:fingerprint"` + LastSeenAt int64 `gorm:"column:last_seen_at"` + Status int `gorm:"column:status"` + Createtime int64 `gorm:"column:createtime"` + Updatetime int64 `gorm:"column:updatetime"` + }{ + UserID: userID, Name: name, Kind: kind, Platform: "darwin", Version: "e2e", + Fingerprint: fingerprint, LastSeenAt: now, Status: 1, Createtime: now, Updatetime: now, + } + if err := tx.Table("devices").Create(&row).Error; err != nil { return nil, fmt.Errorf("insert %s device: %w", kind, err) } + out.DeviceID = row.ID plain, err := randomToken() if err != nil { return nil, err @@ -324,7 +344,7 @@ func seedWorkspace(tx *gorm.DB, userID int64, runID, agentredFP, projectPath str `INSERT INTO sync_objects (user_id, kind, sync_id, project_sync_id, agentred_fingerprint, payload, version, sync_updated_at, source_device_id, deleted_at, createtime, updatetime) - VALUES (?, ?, ?, ?, ?, ?::jsonb, ?, ?, 0, 0, ?, ?)`, + VALUES (?, ?, ?, ?, ?, CAST(? AS JSON), ?, ?, 0, 0, ?, ?)`, userID, kind, syncID, projectSyncID, fingerprint, string(body), version, now, now, now, ).Error; err != nil { return fmt.Errorf("insert sync_object %s/%s: %w", kind, syncID, err) @@ -405,7 +425,7 @@ type cleanupResult struct { func runCleanup(args []string) error { fs := flag.NewFlagSet("cleanup", flag.ExitOnError) - dsn := fs.String("dsn", os.Getenv("WEBE2E_DSN"), "PostgreSQL DSN") + dsn := fs.String("dsn", os.Getenv("WEBE2E_DSN"), "MySQL DSN") runID := fs.String("run-id", "", "run id used at seed time") redisAddr, redisPassword, redisDB := registerRedisFlags(fs) if err := fs.Parse(args); err != nil { diff --git a/go.mod b/go.mod index 1278d593..9ad65d35 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,7 @@ require ( github.com/stretchr/testify v1.11.1 go.uber.org/mock v0.6.0 go.uber.org/zap v1.27.0 - gopkg.in/yaml.v3 v3.0.1 - gorm.io/driver/postgres v1.6.0 + gorm.io/driver/mysql v1.5.7 gorm.io/gorm v1.25.12 ) @@ -59,10 +58,6 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.6.0 // indirect - github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -128,6 +123,6 @@ require ( google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gorm.io/driver/mysql v1.5.7 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/plugin/opentelemetry v0.1.11 // indirect ) diff --git a/go.sum b/go.sum index b0f5784a..74bf3503 100644 --- a/go.sum +++ b/go.sum @@ -105,14 +105,6 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= -github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -343,8 +335,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= -gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= -gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= gorm.io/driver/sqlite v1.5.0 h1:zKYbzRCpBrT1bNijRnxLDJWPjVfImGEn0lSnUY5gZ+c= gorm.io/driver/sqlite v1.5.0/go.mod h1:kDMDfntV9u/vuMmz8APHtHF0b4nyBB7sfCieC6G8k8I= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= diff --git a/internal/controller/device_ctr/register_web_test.go b/internal/controller/device_ctr/register_web_test.go index 47db7837..52b5e72b 100644 --- a/internal/controller/device_ctr/register_web_test.go +++ b/internal/controller/device_ctr/register_web_test.go @@ -50,7 +50,7 @@ func newRegisterWebServer(t *testing.T) *registerWebServer { gin.SetMode(gin.TestMode) testutils.Redis() // miniredis → cago redis.Default() - _, gormDB, mock := hubtest.DatabasePG(t) + _, gormDB, mock := hubtest.Database(t) db.SetDefault(gormDB) signer, err := jwt.NewSigner(testkeys.PrivatePEM, testkeys.PublicPEM, "agentre-server", "agentre") diff --git a/internal/model/entity/device_token_entity/device_token.go b/internal/model/entity/device_token_entity/device_token.go index da1dd22e..d31149a6 100644 --- a/internal/model/entity/device_token_entity/device_token.go +++ b/internal/model/entity/device_token_entity/device_token.go @@ -11,7 +11,7 @@ type DeviceToken struct { RotatedFromID int64 `gorm:"column:rotated_from_id;type:bigint;not null;default:0"` RevokedAt int64 `gorm:"column:revoked_at;type:bigint;not null;default:0"` UserAgent string `gorm:"column:user_agent;type:text;not null;default:''"` - IP *string `gorm:"column:ip;type:inet"` + IP *string `gorm:"column:ip;type:varchar(45)"` Createtime int64 `gorm:"column:createtime;type:bigint;not null;default:0"` } diff --git a/internal/model/entity/sync_entity/sync.go b/internal/model/entity/sync_entity/sync.go index 56bf338a..cadb6d42 100644 --- a/internal/model/entity/sync_entity/sync.go +++ b/internal/model/entity/sync_entity/sync.go @@ -42,7 +42,7 @@ type SyncObject struct { SyncID string `gorm:"column:sync_id;type:text;not null"` ProjectSyncID string `gorm:"column:project_sync_id;type:text;not null;default:''"` AgentredFingerprint string `gorm:"column:agentred_fingerprint;type:text;not null;default:''"` - Payload string `gorm:"column:payload;type:jsonb;not null"` + Payload string `gorm:"column:payload;type:json;not null"` // Version 由账号级单调序列分配，是 R4 唯一的胜负依据。 Version int64 `gorm:"column:version;type:bigint;not null"` // SyncUpdatedAt 是客户端提交的最后修改时间，只用于展示与 30 天窗口计算， @@ -60,7 +60,7 @@ func (*SyncObject) TableName() string { return "sync_objects" } func (o *SyncObject) IsDeleted() bool { return o != nil && o.DeletedAt > 0 } // 账号级单调递增的版本序列住在 sync_account_seqs 表里，这里**刻意没有**对应的 -// entity：分配版本必须是一条 `INSERT … ON CONFLICT DO UPDATE … RETURNING` +// entity：分配版本必须用 `INSERT … ON DUPLICATE KEY UPDATE` // （见 sync_repo.NextVersion），一个 gorm 结构体只会引来先读后写那种用法，而先读 // 后写在多副本并发上行时会把同一个版本号发给两次上行，R4 的「较大者胜」立刻失去 // 可比性。表结构以迁移里的 DDL 为准。 diff --git a/internal/model/entity/user_identity_entity/user_identity.go b/internal/model/entity/user_identity_entity/user_identity.go index 76f3a3de..f1bfcee6 100644 --- a/internal/model/entity/user_identity_entity/user_identity.go +++ b/internal/model/entity/user_identity_entity/user_identity.go @@ -10,7 +10,7 @@ type UserIdentity struct { ProviderUID string `gorm:"column:provider_uid;type:text;not null"` ProviderLogin string `gorm:"column:provider_login;type:text;not null;default:''"` Email string `gorm:"column:email;type:text;not null"` - RawProfile []byte `gorm:"column:raw_profile;type:jsonb;not null;default:'{}'"` + RawProfile []byte `gorm:"column:raw_profile;type:json;not null"` Createtime int64 `gorm:"column:createtime;type:bigint;not null;default:0"` Updatetime int64 `gorm:"column:updatetime;type:bigint;not null;default:0"` } diff --git a/internal/repository/device_flow_repo/device_flow_test.go b/internal/repository/device_flow_repo/device_flow_test.go index d7a15ec5..c04fda56 100644 --- a/internal/repository/device_flow_repo/device_flow_test.go +++ b/internal/repository/device_flow_repo/device_flow_test.go @@ -11,7 +11,7 @@ import ( ) func TestApprove(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( @@ -28,7 +28,7 @@ func TestApprove(t *testing.T) { // 竞败方（行已被别人改过）必须能从返回的行数上看出来。 func TestApprove_ReturnsZeroRowsWhenNothingMatched(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( @@ -44,7 +44,7 @@ func TestApprove_ReturnsZeroRowsWhenNothingMatched(t *testing.T) { } func TestDeny(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( @@ -63,7 +63,7 @@ func TestDeny(t *testing.T) { // 只有 consumed_at=0 的话，用户点「拒绝」的事务先提交、设备的换取事务随后跑这条 // UPDATE，denied_at 已经不为 0 却照样命中 1 行——用户明明拒绝了，设备还是拿到 token。 func TestMarkConsumed_RequiresUnsettledRow(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( @@ -80,7 +80,7 @@ func TestMarkConsumed_RequiresUnsettledRow(t *testing.T) { // 行已被并发请求消费（或拒绝）时 UPDATE 命中 0 行，行数原样透传给 service。 func TestMarkConsumed_ReturnsZeroRowsWhenAlreadySettled(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( @@ -96,7 +96,7 @@ func TestMarkConsumed_ReturnsZeroRowsWhenAlreadySettled(t *testing.T) { } func TestFindByDeviceCode_Found(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceFlow() mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "device_flow_codes" WHERE device_code=$1 ORDER BY "device_flow_codes"."device_code" LIMIT $2`)). WithArgs("dc-x", 1). diff --git a/internal/repository/device_repo/device.go b/internal/repository/device_repo/device.go index 2d3d098e..3f2b2a88 100644 --- a/internal/repository/device_repo/device.go +++ b/internal/repository/device_repo/device.go @@ -5,6 +5,7 @@ import ( "github.com/cago-frame/cago/database/db" "github.com/cago-frame/cago/pkg/consts" + "gorm.io/gorm" "gorm.io/gorm/clause" "agentre-server/internal/model/entity/device_entity" @@ -56,8 +57,8 @@ func (r *repo) FindByFingerprint(ctx context.Context, userID int64, fp string) ( } // Upsert 按 (user_id, fingerprint) 落库：走 uk_devices_user_fingerprint 的 -// ON CONFLICT ... DO UPDATE，一条语句由数据库原子裁决。d.ID 与 d.Createtime 由 -// RETURNING 回填。 +// ON DUPLICATE KEY UPDATE 由数据库原子裁决。MySQL 没有通用的 +// INSERT ... transaction read-back，所以写入后在同一事务内读回最终行填充 d。 // // 不写成「先按 (user_id, fingerprint) 查、再 Save/Create」：那是先查后写，两个已授权的 // device_code 共用同一 (user_id, fingerprint) 并发换取时会双双查空、双双 INSERT， @@ -65,16 +66,23 @@ func (r *repo) FindByFingerprint(ctx context.Context, userID int64, fp string) ( // // createtime 不在赋值列里：命中已有设备时保留它首次注册的时间。 func (r *repo) Upsert(ctx context.Context, d *device_entity.Device) error { - return db.Ctx(ctx).Clauses( - clause.OnConflict{ + return db.Ctx(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "user_id"}, {Name: "fingerprint"}}, DoUpdates: clause.AssignmentColumns([]string{ "name", "kind", "platform", "version", "last_seen_at", "status", "updatetime", }), - }, - clause.Returning{Columns: []clause.Column{{Name: "id"}, {Name: "createtime"}}}, - ).Create(d).Error + }).Create(d).Error; err != nil { + return err + } + final := &device_entity.Device{} + if err := tx.Where("user_id=? AND fingerprint=?", d.UserID, d.Fingerprint).First(final).Error; err != nil { + return err + } + *d = *final + return nil + }) } func (r *repo) Touch(ctx context.Context, id, nowMs int64) error { diff --git a/internal/repository/device_repo/device_test.go b/internal/repository/device_repo/device_test.go index fd326eb8..9bad7a20 100644 --- a/internal/repository/device_repo/device_test.go +++ b/internal/repository/device_repo/device_test.go @@ -14,32 +14,24 @@ import ( // Upsert 必须是一条语句。先 SELECT 再 INSERT/UPDATE 的写法在并发下会双双走到 // INSERT：两个已授权的 device_code 共用同一 (user_id, fingerprint) 同时换取时， // 竞败方撞上 uk_devices_user_fingerprint，拿到的是一个唯一约束错误（映射成 500）， -// 而不是任何约定的 OAuth 错误。ON CONFLICT ... DO UPDATE 由数据库原子裁决， +// 而不是任何约定的 OAuth 错误。ON DUPLICATE KEY UPDATE 由数据库原子裁决， // 两边都拿到同一行、都成功。 -func TestUpsert_SingleStatementOnConflict(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) +func TestUpsert_AtomicWriteThenReadsFinalRow(t *testing.T) { + ctx, _, mock := hubtest.Database(t) r := NewDevice() mock.ExpectBegin() // 赋值列里没有 createtime：命中已有设备时首次注册时间不能被这次换取的 nowMs 抹掉。 - mock.ExpectQuery(regexp.QuoteMeta( - `ON CONFLICT ("user_id","fingerprint") DO UPDATE SET `+ - `"name"="excluded"."name",`+ - `"kind"="excluded"."kind",`+ - `"platform"="excluded"."platform",`+ - `"version"="excluded"."version",`+ - `"last_seen_at"="excluded"."last_seen_at",`+ - `"status"="excluded"."status",`+ - `"updatetime"="excluded"."updatetime" `+ - `RETURNING "id","createtime"`, - )).WillReturnRows( - sqlmock.NewRows([]string{"id", "createtime"}).AddRow(int64(100), int64(1000)), - ) + mock.ExpectExec(regexp.QuoteMeta("ON DUPLICATE KEY UPDATE")). + WillReturnResult(sqlmock.NewResult(100, 2)) + mock.ExpectQuery("SELECT \\* FROM `devices` WHERE user_id=\\? AND fingerprint=\\?"). + WillReturnRows(sqlmock.NewRows([]string{"id", "user_id", "fingerprint", "createtime"}). + AddRow(int64(100), int64(7), "fp-new", int64(1000))) mock.ExpectCommit() d := &device_entity.Device{UserID: 7, Fingerprint: "fp-new", Kind: "agentred", Status: 1, Createtime: 2000} assert.NoError(t, r.Upsert(ctx, d)) - // RETURNING 把库里那一行回填进实体：命中已有设备时拿到的是它原来的 id 和 createtime。 + // 事务内读回最终行：命中已有设备时拿到的是它原来的 id 和 createtime。 assert.Equal(t, int64(100), d.ID) assert.Equal(t, int64(1000), d.Createtime) assert.NoError(t, mock.ExpectationsWereMet()) diff --git a/internal/repository/device_token_repo/device_token_test.go b/internal/repository/device_token_repo/device_token_test.go index 7caeb1c9..bd3feb59 100644 --- a/internal/repository/device_token_repo/device_token_test.go +++ b/internal/repository/device_token_repo/device_token_test.go @@ -12,7 +12,7 @@ import ( ) func TestRevokeChain(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() mock.ExpectBegin() @@ -26,7 +26,7 @@ func TestRevokeChain(t *testing.T) { } func TestRevoke_RequiresUnrevokedRow(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() mock.ExpectBegin() @@ -43,7 +43,7 @@ func TestRevoke_RequiresUnrevokedRow(t *testing.T) { // 行已被并发请求轮换时 UPDATE 命中 0 行，行数原样透传给 service。 func TestRevoke_ReturnsZeroRowsWhenAlreadyRevoked(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() mock.ExpectBegin() @@ -59,7 +59,7 @@ func TestRevoke_ReturnsZeroRowsWhenAlreadyRevoked(t *testing.T) { } func TestFindByHash_Found(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "device_tokens" WHERE refresh_token_hash=$1 ORDER BY "device_tokens"."id" LIMIT $2`)). WithArgs("h1", 1). @@ -71,18 +71,18 @@ func TestFindByHash_Found(t *testing.T) { } func TestCreate(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() mock.ExpectBegin() // R4 整条链路都挂在 access_jti 真的被写进去上（Revoke 拉黑它、吊销列表分发它）。 // 十个 AnyArg 的期望连列名都不看，删掉 AccessJTI 字段照样绿，所以这里把列名和 // 那一列的值都钉死。 - mock.ExpectQuery(regexp.QuoteMeta( + mock.ExpectExec(regexp.QuoteMeta( `INSERT INTO "device_tokens" ("device_id","refresh_token_hash","access_jti"`)). WithArgs(int64(42), "h", "jti-1", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). - WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(int64(99))) + WillReturnResult(sqlmock.NewResult(99, 1)) mock.ExpectCommit() e := &device_token_entity.DeviceToken{DeviceID: 42, RefreshTokenHash: "h", RefreshExpiresAt: 1000, AccessJTI: "jti-1"} assert.NoError(t, r.Create(ctx, e)) @@ -90,7 +90,7 @@ func TestCreate(t *testing.T) { } func TestListAccessJTIByDevice(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() mock.ExpectQuery(regexp.QuoteMeta(`SELECT "access_jti" FROM "device_tokens" WHERE device_id=$1 AND access_jti != ''`)). WithArgs(int64(42)). @@ -102,7 +102,7 @@ func TestListAccessJTIByDevice(t *testing.T) { } func TestListRevokedJTIByUser(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() mock.ExpectQuery(regexp.QuoteMeta( `SELECT "device_tokens"."access_jti" FROM "device_tokens" JOIN devices ON devices.id = device_tokens.device_id WHERE devices.user_id = $1 AND device_tokens.revoked_at != 0 AND device_tokens.access_jti != '' AND device_tokens.createtime >= $2`)). @@ -117,7 +117,7 @@ func TestListRevokedJTIByUser(t *testing.T) { // 同一条 device_tokens 行既是 Revoke() 的写入目标，也是本查询的读取来源： // 撤销与分发之间没有缓存/队列，写入即刻可读——这是 R4「reflects a Revoke immediately」成立的机制。 func TestListRevokedJTIByUser_ReflectsRevokeImmediately(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() mock.ExpectBegin() @@ -140,7 +140,7 @@ func TestListRevokedJTIByUser_ReflectsRevokeImmediately(t *testing.T) { } func TestListRevokedJTIByUser_ExcludesOutsideAccessTTLWindow(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() // windowStartMs 之前签发的行已被数据库端的 createtime >= ? 条件排除， // mock 只按预期 SQL 返回窗口内的一行——验证调用方传入的 windowStartMs 确实被当成查询条件。 diff --git a/internal/repository/follow_repo/follow.go b/internal/repository/follow_repo/follow.go index f76c333c..a3a482c1 100644 --- a/internal/repository/follow_repo/follow.go +++ b/internal/repository/follow_repo/follow.go @@ -31,7 +31,7 @@ func NewFollow() FollowRepo { return &repo{} } type repo struct{} -// Follow 是一条语句的条件插入：ON CONFLICT ... DO NOTHING 由数据库原子裁决， +// Follow 是一条语句的条件插入：ON DUPLICATE KEY ... DO NOTHING 由数据库原子裁决， // 并发重复关注两边都成功、只落一行。先查再插会在两个副本同时首次关注时双双 // 走到 INSERT，竞败方撞唯一索引拿到一个约束错误。 func (r *repo) Follow(ctx context.Context, f *follow_entity.FollowedSession) error { diff --git a/internal/repository/follow_repo/follow_test.go b/internal/repository/follow_repo/follow_test.go index 121230bf..53349204 100644 --- a/internal/repository/follow_repo/follow_test.go +++ b/internal/repository/follow_repo/follow_test.go @@ -12,21 +12,21 @@ import ( hubtest "agentre-server/internal/testutils" ) -// Follow 必须是一条语句：INSERT ... ON CONFLICT (user_id, device_fingerprint, +// Follow 必须是一条语句：INSERT ... ON DUPLICATE KEY (user_id, device_fingerprint, // session_id) DO NOTHING。重复关注（同账号、同一目标会话）命中唯一索引时是 // no-op，由数据库原子裁决：不新增行、也不重置首次关注时间——R12「关注幂等」在 // 数据层的落点。这里用 0 行受影响的结果模拟「已关注」的那次重复请求。 func TestFollow_SingleStatementOnConflictDoNothing(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewFollow() mock.ExpectBegin() // 绑定值也一并钉住：少绑一列（比如把 user_id 漏在 WHERE 之外）时唯一索引的 // 裁决对象就变了，只验 SQL 文本看不出来。 - mock.ExpectQuery(regexp.QuoteMeta( - `ON CONFLICT ("user_id","device_fingerprint","session_id") DO NOTHING`, + mock.ExpectExec(regexp.QuoteMeta( + `ON DUPLICATE KEY UPDATE`, )).WithArgs(int64(7), "fp-daemon-1", "sess-9", int64(1000), int64(1000), int64(1000)). - WillReturnRows(sqlmock.NewRows([]string{"id"})) // 空结果 = 已关注，冲突 no-op + WillReturnResult(sqlmock.NewResult(0, 0)) // 0 行 = 已关注，冲突 no-op mock.ExpectCommit() f := &follow_entity.FollowedSession{ @@ -39,7 +39,7 @@ func TestFollow_SingleStatementOnConflictDoNothing(t *testing.T) { // Unfollow 是一条 DELETE：从未关注 / 已取消时删不到行，仍是成功——R12「取消幂等」。 func TestUnfollow_DeleteIsIdempotent(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewFollow() mock.ExpectBegin() @@ -55,7 +55,7 @@ func TestUnfollow_DeleteIsIdempotent(t *testing.T) { // ListByUser 只按账号过滤：名单属于账号，不属于某一台设备或某一个浏览器（R14）。 // 不按在线态过滤——机器离线时该条仍在名单里（R13）。 func TestListByUser_AccountScoped(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewFollow() rows := sqlmock.NewRows([]string{ diff --git a/internal/repository/sync_repo/avatar.go b/internal/repository/sync_repo/avatar.go index 404207db..111fad6e 100644 --- a/internal/repository/sync_repo/avatar.go +++ b/internal/repository/sync_repo/avatar.go @@ -69,7 +69,7 @@ WHERE a.createtime < ? WHERE o.user_id = a.user_id AND o.kind = 'agent' AND o.deleted_at = 0 - AND o.payload->>'avatar_hash' = a.content_hash + AND JSON_UNQUOTE(JSON_EXTRACT(o.payload, '$.avatar_hash')) = a.content_hash )` func (r *avatarRepo) DeleteUnreferencedBefore(ctx context.Context, cutoff int64) (int64, error) { diff --git a/internal/repository/sync_repo/object.go b/internal/repository/sync_repo/object.go index 146fa389..fbc8fe1d 100644 --- a/internal/repository/sync_repo/object.go +++ b/internal/repository/sync_repo/object.go @@ -5,6 +5,7 @@ import ( "context" "github.com/cago-frame/cago/database/db" + "gorm.io/gorm" "gorm.io/gorm/clause" "agentre-server/internal/model/entity/sync_entity" @@ -79,16 +80,18 @@ func (r *objectRepo) FindLocationByNaturalKey( // 写入才覆盖。多副本并发上行同一行时由数据库裁决，先查后写会让落后的那次把更新 // 的那一版盖掉。createtime 不在赋值列里：命中已有行时保留它首次落地的时间。 func (r *objectRepo) Save(ctx context.Context, obj *sync_entity.SyncObject) error { + assignments := map[string]interface{}{} + for _, column := range []string{ + "kind", "project_sync_id", "agentred_fingerprint", "payload", + "version", "sync_updated_at", "source_device_id", "deleted_at", "updatetime", + } { + assignments[column] = gorm.Expr( + "IF(VALUES(`version`) > `version`, VALUES(`" + column + "`), `" + column + "`)", + ) + } return db.Ctx(ctx).Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "user_id"}, {Name: "sync_id"}}, - DoUpdates: clause.AssignmentColumns([]string{ - "kind", "project_sync_id", "agentred_fingerprint", "payload", - "version", "sync_updated_at", "source_device_id", "deleted_at", "updatetime", - }), - Where: clause.Where{Exprs: []clause.Expression{clause.Lt{ - Column: clause.Column{Table: "sync_objects", Name: "version"}, - Value: clause.Column{Table: "excluded", Name: "version"}, - }}}, + Columns: []clause.Column{{Name: "user_id"}, {Name: "sync_id"}}, + DoUpdates: clause.Assignments(assignments), }).Create(obj).Error } diff --git a/internal/repository/sync_repo/state.go b/internal/repository/sync_repo/state.go index 346ce9c9..2bc136dd 100644 --- a/internal/repository/sync_repo/state.go +++ b/internal/repository/sync_repo/state.go @@ -5,6 +5,7 @@ import ( "time" "github.com/cago-frame/cago/database/db" + "gorm.io/gorm" "gorm.io/gorm/clause" "agentre-server/internal/model/entity/sync_entity" @@ -33,17 +34,22 @@ type stateRepo struct{} // NextVersion 必须是一条语句。先读后写在多副本并发上行时会双双读到同一个值、 // 两次上行拿到同一个版本号，R4 的「较大者胜」立刻失去可比性；INSERT … ON -// CONFLICT DO UPDATE … RETURNING 把递增与取值合成一次，由数据库的行锁串行化。 +// CONFLICT DO UPDATE … transaction read-back 把递增与取值合成一次，由数据库的行锁串行化。 func (r *stateRepo) NextVersion(ctx context.Context, userID int64, n int64) (int64, error) { if n <= 0 { n = 1 } now := time.Now().UnixMilli() var version int64 - err := db.Ctx(ctx).Raw(`INSERT INTO sync_account_seqs (user_id, version_seq, updatetime) -VALUES (?, ?, ?) -ON CONFLICT (user_id) DO UPDATE SET version_seq = sync_account_seqs.version_seq + ?, updatetime = ? -RETURNING version_seq`, userID, n, now, n, now).Scan(&version).Error + err := db.Ctx(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Exec(`INSERT INTO sync_account_seqs (user_id, version_seq, updatetime) +VALUES (?, LAST_INSERT_ID(?), ?) +ON DUPLICATE KEY UPDATE version_seq = LAST_INSERT_ID(version_seq + ?), updatetime = ?`, + userID, n, now, n, now).Error; err != nil { + return err + } + return tx.Raw("SELECT LAST_INSERT_ID()").Scan(&version).Error + }) if err != nil { return 0, err } diff --git a/internal/repository/sync_repo/sync_test.go b/internal/repository/sync_repo/sync_test.go index df3e68b8..3d9bfa14 100644 --- a/internal/repository/sync_repo/sync_test.go +++ b/internal/repository/sync_repo/sync_test.go @@ -14,11 +14,14 @@ import ( // 版本序列必须是一条语句：多副本并发上行时，先读后写会双双读到同一个值， // 两次上行拿到同一个版本号，R4 的「较大者胜」立刻失去可比性。 func TestNextVersion_GivenConcurrentReplicas_ThenSingleAtomicStatement(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncState() - mock.ExpectQuery(regexp.QuoteMeta(`INSERT INTO sync_account_seqs`)). - WillReturnRows(sqlmock.NewRows([]string{"version_seq"}).AddRow(int64(42))) + mock.ExpectBegin() + mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO sync_account_seqs`)).WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT LAST_INSERT_ID()`)). + WillReturnRows(sqlmock.NewRows([]string{"LAST_INSERT_ID()"}).AddRow(int64(42))) + mock.ExpectCommit() v, err := r.NextVersion(ctx, 7, 1) assert.NoError(t, err) @@ -27,12 +30,16 @@ func TestNextVersion_GivenConcurrentReplicas_ThenSingleAtomicStatement(t *testin } func TestNextVersion_GivenBatch_ThenTakesNAtOnce(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncState() - mock.ExpectQuery(regexp.QuoteMeta(`ON CONFLICT`)). + mock.ExpectBegin() + mock.ExpectExec(regexp.QuoteMeta(`ON DUPLICATE KEY UPDATE`)). WithArgs(int64(7), int64(3), sqlmock.AnyArg(), int64(3), sqlmock.AnyArg()). - WillReturnRows(sqlmock.NewRows([]string{"version_seq"}).AddRow(int64(45))) + WillReturnResult(sqlmock.NewResult(0, 2)) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT LAST_INSERT_ID()`)). + WillReturnRows(sqlmock.NewRows([]string{"LAST_INSERT_ID()"}).AddRow(int64(45))) + mock.ExpectCommit() v, err := r.NextVersion(ctx, 7, 3) assert.NoError(t, err) @@ -43,7 +50,7 @@ func TestNextVersion_GivenBatch_ThenTakesNAtOnce(t *testing.T) { // 没有 last_sync_at 记录 = 首次登录，仓储层返回 (nil, nil) 而不是错误， // R6a 的「不算超窗口」才判得出来。 func TestFindDeviceState_GivenNoRow_ThenNilNil(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncState() mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "sync_device_states"`)). @@ -58,11 +65,11 @@ func TestFindDeviceState_GivenNoRow_ThenNilNil(t *testing.T) { // last_sync_at 也是一条语句的 upsert：同一台设备的上行与下行会并发落到不同副本， // 先查后写会漏掉其中一次。 func TestTouchDeviceState_GivenNoRow_ThenUpsertsInOneStatement(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncState() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`ON CONFLICT ("user_id","device_id") DO UPDATE SET`)). + mock.ExpectExec(regexp.QuoteMeta(`ON DUPLICATE KEY UPDATE`)). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectCommit() @@ -73,12 +80,11 @@ func TestTouchDeviceState_GivenNoRow_ThenUpsertsInOneStatement(t *testing.T) { // 落库要在版本号更大时才覆盖：两个副本并发写同一行时由数据库裁决， // 落后的那次不能把更新的那一版盖掉。 func TestSaveObject_GivenExistingRow_ThenUpsertOnlyWhenVersionIsGreater(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncObject() mock.ExpectBegin() - mock.ExpectQuery(regexp.QuoteMeta(`ON CONFLICT ("user_id","sync_id") DO UPDATE SET`)). - WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(int64(1))) + mock.ExpectExec(regexp.QuoteMeta(`ON DUPLICATE KEY UPDATE`)).WillReturnResult(sqlmock.NewResult(1, 2)) mock.ExpectCommit() err := r.Save(ctx, &sync_entity.SyncObject{ @@ -89,12 +95,12 @@ func TestSaveObject_GivenExistingRow_ThenUpsertOnlyWhenVersionIsGreater(t *testi } func TestSaveObject_GivenExistingRow_ThenConditionalWhereIsPresent(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncObject() mock.ExpectBegin() - mock.ExpectQuery(regexp.QuoteMeta(`WHERE "sync_objects"."version" < "excluded"."version"`)). - WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(int64(1))) + mock.ExpectExec("`version`=IF\\(VALUES\\(`version`\\) > `version`, VALUES\\(`version`\\), `version`\\)"). + WillReturnResult(sqlmock.NewResult(1, 2)) mock.ExpectCommit() assert.NoError(t, r.Save(ctx, &sync_entity.SyncObject{ @@ -105,7 +111,7 @@ func TestSaveObject_GivenExistingRow_ThenConditionalWhereIsPresent(t *testing.T) // 自然键查重只看存活的那一行：墓碑不占（账号, 项目, 指纹），否则删掉再建就建不回来。 func TestFindLocationByNaturalKey_GivenTombstones_ThenOnlyLiveRowMatches(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncObject() mock.ExpectQuery(regexp.QuoteMeta(`deleted_at=0`)). @@ -120,7 +126,7 @@ func TestFindLocationByNaturalKey_GivenTombstones_ThenOnlyLiveRowMatches(t *test // 打墓碑是条件更新，返回受影响行数：已经是墓碑时为 0，由 service 决定这意味着什么。 func TestTombstone_GivenAlreadyTombstoned_ThenZeroRowsAffected(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncObject() mock.ExpectBegin() @@ -136,7 +142,7 @@ func TestTombstone_GivenAlreadyTombstoned_ThenZeroRowsAffected(t *testing.T) { // 下行游标：版本升序、严格大于游标、按 limit 截断。 func TestListSince_GivenCursor_ThenOrderedByVersionAscending(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncObject() mock.ExpectQuery(regexp.QuoteMeta(`ORDER BY version ASC`)). @@ -156,7 +162,7 @@ func TestListSince_GivenCursor_ThenOrderedByVersionAscending(t *testing.T) { // 这条路径服务的是「总览页列 Agent」「设备展开列项目」，需要的是当前状态的 // 完整集合，不是增量。墓碑必须被过滤掉，否则已删除的 Agent 会在总览重新出现。 func TestListByKinds_GivenKinds_ThenOnlyLiveRowsOfThoseKinds(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncObject() mock.ExpectQuery(regexp.QuoteMeta(`AND kind IN (`)+`.*`+regexp.QuoteMeta(`) AND deleted_at=0`)). @@ -174,7 +180,7 @@ func TestListByKinds_GivenKinds_ThenOnlyLiveRowsOfThoseKinds(t *testing.T) { // 上报组整份替换：先清掉这台设备的旧清单，再写新的，且两步在同一个事务里， // 否则中途失败会让服务端的清单空掉。 func TestReplaceSnapshot_GivenItems_ThenDeletesThenInsertsInOneTransaction(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncLocalPath() mock.ExpectBegin() @@ -192,7 +198,7 @@ func TestReplaceSnapshot_GivenItems_ThenDeletesThenInsertsInOneTransaction(t *te } func TestReplaceSnapshot_GivenEmptySnapshot_ThenOnlyDeletes(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncLocalPath() mock.ExpectBegin() @@ -207,7 +213,7 @@ func TestReplaceSnapshot_GivenEmptySnapshot_ThenOnlyDeletes(t *testing.T) { // R18：设备记录被删除时，它上报的本机路径清单一并消失。device_id 全局唯一， // 不需要再传 user_id 校验归属，也不是一个事务（单条 DELETE，没有后续要写的东西）。 func TestDeleteByDevice_GivenDeviceID_ThenDeletesAllRowsForThatDevice(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncLocalPath() mock.ExpectBegin() @@ -223,7 +229,7 @@ func TestDeleteByDevice_GivenDeviceID_ThenDeletesAllRowsForThatDevice(t *testing // 设备展开页要知道「这台设备上配了路径的项目有哪些」——只看这台设备名下的 // 上报行，不看它的正文（web 端不展示路径，R19），这里只验证按设备取全部行。 func TestListByDevice_GivenDeviceID_ThenReturnsItsReportedRows(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncLocalPath() mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "device_local_paths" WHERE user_id=`)). @@ -240,11 +246,11 @@ func TestListByDevice_GivenDeviceID_ThenReturnsItsReportedRows(t *testing.T) { // 同一份头像重复上传不该产生第二行，也不该覆盖已有正文。 func TestSaveAvatar_GivenSameContentTwice_ThenOnConflictDoNothing(t *testing.T) { - ctx, _, mock := hubtest.DatabasePG(t) + ctx, _, mock := hubtest.Database(t) r := NewSyncAvatar() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`ON CONFLICT DO NOTHING`)). + mock.ExpectExec(regexp.QuoteMeta(`ON DUPLICATE KEY UPDATE`)). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() @@ -258,7 +264,7 @@ func TestSaveAvatar_GivenSameContentTwice_ThenOnConflictDoNothing(t *testing.T) // 没有 deleted_at>0 就把存活的行一起删了；没有 deleted_at ''; - + updatetime bigint NOT NULL DEFAULT 0, + live_location_key varchar(511) GENERATED ALWAYS AS + (IF(kind = 'project_location' AND deleted_at = 0, + CONCAT(project_sync_id, CHAR(0), agentred_fingerprint), NULL)) STORED, + UNIQUE KEY uk_sync_objects_identity (user_id, sync_id), + UNIQUE KEY uk_sync_objects_location (user_id, live_location_key), + KEY idx_sync_objects_cursor (user_id, version), + KEY idx_sync_objects_fingerprint (user_id, agentred_fingerprint) + )`, ` CREATE TABLE sync_account_seqs ( user_id bigint PRIMARY KEY, version_seq bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0 - ); - + )`, ` CREATE TABLE sync_device_states ( user_id bigint NOT NULL, device_id bigint NOT NULL, last_sync_at bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0, PRIMARY KEY (user_id, device_id) - ); - + )`, ` CREATE TABLE sync_avatars ( user_id bigint NOT NULL, - content_hash text NOT NULL, - content_type text NOT NULL DEFAULT '', + content_hash varchar(64) NOT NULL, + content_type varchar(255) NOT NULL DEFAULT '', content text NOT NULL, byte_size bigint NOT NULL DEFAULT 0, createtime bigint NOT NULL DEFAULT 0, PRIMARY KEY (user_id, content_hash) - ); - + )`, ` CREATE TABLE device_local_paths ( user_id bigint NOT NULL, device_id bigint NOT NULL, - project_sync_id text NOT NULL, + project_sync_id varchar(255) NOT NULL, path text NOT NULL, updatetime bigint NOT NULL DEFAULT 0, PRIMARY KEY (user_id, device_id, project_sync_id) - ); - `).Error + )`, + } + for _, statement := range statements { + if err := tx.Exec(statement).Error; err != nil { + return err + } + } + return nil }, Rollback: func(tx *gorm.DB) error { - return tx.Exec(` - DROP TABLE IF EXISTS device_local_paths; - DROP TABLE IF EXISTS sync_avatars; - DROP TABLE IF EXISTS sync_device_states; - DROP TABLE IF EXISTS sync_account_seqs; - DROP TABLE IF EXISTS sync_objects; - `).Error + for _, table := range []string{ + "device_local_paths", "sync_avatars", "sync_device_states", + "sync_account_seqs", "sync_objects", + } { + if err := tx.Exec("DROP TABLE IF EXISTS " + table).Error; err != nil { + return err + } + } + return nil }, } } diff --git a/migrations/202608100001_followed_sessions.go b/migrations/202608100001_followed_sessions.go index 518d89d5..39eada81 100644 --- a/migrations/202608100001_followed_sessions.go +++ b/migrations/202608100001_followed_sessions.go @@ -10,7 +10,7 @@ import ( // 这是 agentre-server 本轮唯一的服务端新增表，也是硬不变量（server 不持有任何 // 会话内容）的唯一例外，且它存的是「指向」——目标设备指纹 + 会话标识 + 关注时间， // 不含标题、消息或转录。表按 (user_id, device_fingerprint, session_id) 唯一， -// 关注/取消因此幂等：重复关注命中唯一索引时 ON CONFLICT DO NOTHING，不新增行、 +// 关注/取消因此幂等：重复关注命中唯一索引时 ON DUPLICATE KEY DO NOTHING，不新增行、 // 不重置首次关注时间；取消就是一条 DELETE，删不到也是成功。 func migration202608100001() *gormigrate.Migration { return &gormigrate.Migration{ @@ -18,16 +18,16 @@ func migration202608100001() *gormigrate.Migration { Migrate: func(tx *gorm.DB) error { return tx.Exec(` CREATE TABLE followed_sessions ( - id bigserial PRIMARY KEY, + id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, user_id bigint NOT NULL, - device_fingerprint text NOT NULL, - session_id text NOT NULL, + device_fingerprint varchar(255) NOT NULL, + session_id varchar(255) NOT NULL, followed_at bigint NOT NULL DEFAULT 0, createtime bigint NOT NULL DEFAULT 0, - updatetime bigint NOT NULL DEFAULT 0 + updatetime bigint NOT NULL DEFAULT 0, + UNIQUE KEY uk_followed_sessions_identity + (user_id, device_fingerprint, session_id) ); - CREATE UNIQUE INDEX uk_followed_sessions_identity - ON followed_sessions(user_id, device_fingerprint, session_id); `).Error }, Rollback: func(tx *gorm.DB) error { diff --git a/migrations/migrations.go b/migrations/migrations.go index 828fa9ea..84a1203e 100644 --- a/migrations/migrations.go +++ b/migrations/migrations.go @@ -1,4 +1,4 @@ -// Package migrations 汇总并执行 agentre-server PostgreSQL 全部迁移。 +// Package migrations 汇总并执行 agentre-server MySQL 全部迁移。 // // 规范： // - 文件名前缀 = 时间戳排序键（YYYYMMDDNNNN），调用顺序按时间升序。 @@ -20,14 +20,11 @@ import ( "gorm.io/gorm" ) -// migrationLockID 是迁移串行化用的 PostgreSQL advisory lock id。 -// -// 取值 = crc32.ChecksumIEEE([]byte("agentre-server/migrations"))，算好后写死在这里， -// 避免每次启动都重算。advisory lock 只按「当前数据库」隔离、不绑定任何表或行，因此只有 -// 另一个连到同一个数据库的应用凑巧选中同一个 id 才会撞车。 -const migrationLockID int64 = 4097874731 +// migrationLockName 是迁移串行化用的 MySQL named lock。锁名携应用名， +// 避免同一 MySQL 实例上的其他应用迁移相互阻塞。 +const migrationLockName = "agentre-server/migrations" -// migrationLockPollInterval / migrationLockWaitBudget 是轮询 pg_try_advisory_lock 的 +// migrationLockPollInterval / migrationLockWaitBudget 是轮询 GET_LOCK 的 // 间隔与总预算，声明成变量是为了让测试能换成极小值而不必真的等待。 var ( migrationLockPollInterval = 500 * time.Millisecond @@ -36,7 +33,7 @@ var ( // RunMigrations 执行所有迁移。 // -// 副本可能并发启动：先在一条专用连接上取 advisory lock 把迁移串行化，拿到锁的副本才跑 +// 副本可能并发启动：先在一条专用连接上取 named lock 把迁移串行化，拿到锁的副本才跑 // gormigrate，其余副本轮询等待；等待超过 migrationLockWaitBudget 就放弃并报错，交给 // main.go 的 log.Fatalf 退出，由 k8s 重启后重试。 func RunMigrations(db *gorm.DB) error { @@ -48,11 +45,11 @@ func RunMigrations(db *gorm.DB) error { // withMigrationLock 在持有迁移 advisory lock 期间执行 fn。 // -// advisory lock 是会话级的，而 gorm 的 *gorm.DB 从连接池里取连接：如果直接用 -// db.Exec("pg_try_advisory_lock") 加锁，锁可能落在连接 A 上，随后 fn 里的迁移语句却从 +// named lock 是会话级的，而 gorm 的 *gorm.DB 从连接池里取连接：如果直接用 +// db.Exec("GET_LOCK") 加锁，锁可能落在连接 A 上，随后 fn 里的迁移语句却从 // 连接 B 跑，锁在归还连接池的那一刻就形同虚设。因此这里用 sqlDB.Conn(ctx) 单独要一条 // 连接、全程只用它做加锁/解锁；fn 内部的迁移仍然照常走连接池，advisory lock 不绑定数据， -// 这样是安全的。副本崩溃时这条专用连接断开，PostgreSQL 会自动释放锁，无需人工介入。 +// 这样是安全的。副本崩溃时这条专用连接断开，MySQL 会自动释放锁，无需人工介入。 func withMigrationLock(db *gorm.DB, fn func() error) error { sqlDB, err := db.DB() if err != nil { @@ -74,33 +71,33 @@ func withMigrationLock(db *gorm.DB, fn func() error) error { return fn() } -// acquireMigrationLock 轮询 pg_try_advisory_lock 直到拿到锁或等待预算耗尽。 +// acquireMigrationLock 轮询 GET_LOCK 直到拿到锁或等待预算耗尽。 // -// 用轮询而非阻塞的 pg_advisory_lock，是为了让等待有明确上界、可观测：拿不到锁时立刻 +// 用零超时 GET_LOCK 轮询，是为了让等待有明确上界、可观测：拿不到锁时立刻 // 返回，不会让副本静默挂起到被存活探针杀掉。 func acquireMigrationLock(ctx context.Context, conn *sql.Conn) error { deadline := time.Now().Add(migrationLockWaitBudget) for { - var locked bool - if err := conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", migrationLockID).Scan(&locked); err != nil { - return fmt.Errorf("migrations: try advisory lock: %w", err) + var locked int + if err := conn.QueryRowContext(ctx, "SELECT GET_LOCK(?, 0)", migrationLockName).Scan(&locked); err != nil { + return fmt.Errorf("migrations: try named lock: %w", err) } - if locked { + if locked == 1 { return nil } if !time.Now().Before(deadline) { - return fmt.Errorf("migrations: timed out after %s waiting for the migration lock (id %d); another replica is likely still migrating", - migrationLockWaitBudget, migrationLockID) + return fmt.Errorf("migrations: timed out after %s waiting for the migration lock %q; another replica is likely still migrating", + migrationLockWaitBudget, migrationLockName) } time.Sleep(migrationLockPollInterval) } } -// releaseMigrationLock 解锁；专用连接紧接着就会被关闭，即便这里失败，PostgreSQL 也会在 +// releaseMigrationLock 解锁；专用连接紧接着就会被关闭，即便这里失败，MySQL 也会在 // 连接断开时释放这条会话级的锁，因此这里只记日志、不把错误抛给调用方。 func releaseMigrationLock(ctx context.Context, conn *sql.Conn) { - var unlocked bool - if err := conn.QueryRowContext(ctx, "SELECT pg_advisory_unlock($1)", migrationLockID).Scan(&unlocked); err != nil { + var unlocked int + if err := conn.QueryRowContext(ctx, "SELECT RELEASE_LOCK(?)", migrationLockName).Scan(&unlocked); err != nil { logger.Ctx(ctx).Warn("release migration lock", zap.Error(err)) } } diff --git a/migrations/migrations_test.go b/migrations/migrations_test.go deleted file mode 100644 index fc367808..00000000 --- a/migrations/migrations_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package migrations - -import ( - "regexp" - "testing" - "time" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/stretchr/testify/assert" - - hubtest "agentre-server/internal/testutils" -) - -// withPatchedLockTiming 把轮询间隔和等待预算换成测试专用的小值，跑完自动还原， -// 避免真跑迁移锁默认的 120s 预算拖慢测试。 -func withPatchedLockTiming(t *testing.T, interval, budget time.Duration) { - t.Helper() - origInterval, origBudget := migrationLockPollInterval, migrationLockWaitBudget - migrationLockPollInterval, migrationLockWaitBudget = interval, budget - t.Cleanup(func() { - migrationLockPollInterval, migrationLockWaitBudget = origInterval, origBudget - }) -} - -// TestWithMigrationLock_RetriesUntilAcquired 断言：拿不到锁时会重试（pg_try_advisory_lock -// 返回 false），拿到锁（返回 true）后才跑传入的迁移函数，结束后释放锁（pg_advisory_unlock）。 -func TestWithMigrationLock_RetriesUntilAcquired(t *testing.T) { - withPatchedLockTiming(t, time.Millisecond, 120*time.Second) - _, gormDB, mock := hubtest.DatabasePG(t) - - mock.ExpectQuery(regexp.QuoteMeta("pg_try_advisory_lock")). - WillReturnRows(sqlmock.NewRows([]string{"pg_try_advisory_lock"}).AddRow(false)) - mock.ExpectQuery(regexp.QuoteMeta("pg_try_advisory_lock")). - WillReturnRows(sqlmock.NewRows([]string{"pg_try_advisory_lock"}).AddRow(true)) - mock.ExpectQuery(regexp.QuoteMeta("pg_advisory_unlock")). - WillReturnRows(sqlmock.NewRows([]string{"pg_advisory_unlock"}).AddRow(true)) - - ran := false - err := withMigrationLock(gormDB, func() error { - ran = true - return nil - }) - - assert.NoError(t, err) - assert.True(t, ran, "migration func should run once the lock is acquired") - assert.NoError(t, mock.ExpectationsWereMet()) -} - -// TestWithMigrationLock_PropagatesMigrateError 断言：迁移函数本身的失败会被如实返回， -// 但锁依然会在结束时释放（不会把连接晾在持锁状态）。 -func TestWithMigrationLock_PropagatesMigrateError(t *testing.T) { - withPatchedLockTiming(t, time.Millisecond, 120*time.Second) - _, gormDB, mock := hubtest.DatabasePG(t) - - mock.ExpectQuery(regexp.QuoteMeta("pg_try_advisory_lock")). - WillReturnRows(sqlmock.NewRows([]string{"pg_try_advisory_lock"}).AddRow(true)) - mock.ExpectQuery(regexp.QuoteMeta("pg_advisory_unlock")). - WillReturnRows(sqlmock.NewRows([]string{"pg_advisory_unlock"}).AddRow(true)) - - wantErr := assert.AnError - err := withMigrationLock(gormDB, func() error { - return wantErr - }) - - assert.ErrorIs(t, err, wantErr) - assert.NoError(t, mock.ExpectationsWereMet()) -} - -// TestWithMigrationLock_TimesOut 断言：轮询超过等待预算后返回一个说明「等迁移锁超时」的 -// 错误，且从未拿到锁，因此从不会调用传入的迁移函数，也不会尝试解锁。 -func TestWithMigrationLock_TimesOut(t *testing.T) { - const ( - interval = 5 * time.Millisecond - budget = 20 * time.Millisecond - ) - withPatchedLockTiming(t, interval, budget) - _, gormDB, mock := hubtest.DatabasePG(t) - - // 预算/间隔比很小，实际重试次数以真实时钟为准，注册一批足够多的「未拿到锁」响应。 - for i := 0; i < 50; i++ { - mock.ExpectQuery(regexp.QuoteMeta("pg_try_advisory_lock")). - WillReturnRows(sqlmock.NewRows([]string{"pg_try_advisory_lock"}).AddRow(false)) - } - - start := time.Now() - ran := false - err := withMigrationLock(gormDB, func() error { - ran = true - return nil - }) - elapsed := time.Since(start) - - assert.Error(t, err) - assert.Contains(t, err.Error(), "timed out") - assert.Contains(t, err.Error(), "migration lock") - assert.False(t, ran, "migration func must not run without the lock") - assert.GreaterOrEqual(t, elapsed, budget, "must have actually waited out the budget via retries") -} From 6a161e48b2e3974c479df3208b3d2e846020f493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E4=B8=80=E4=B9=8B?= Date: Thu, 13 Aug 2026 11:16:17 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20database:=20MySQL=20?= =?UTF-8?q?=E5=9F=BA=E7=BA=BF=E6=8C=89=E5=8E=9F=E7=94=9F=E5=86=99=E6=B3=95?= =?UTF-8?q?=E9=87=8D=E6=9E=84=EF=BC=8C=E4=BF=AE=E6=8E=89=E8=87=AA=E7=84=B6?= =?UTF-8?q?=E9=94=AE=E8=AF=AF=E6=92=9E=E3=80=81=E5=A4=B4=E5=83=8F=2064KB?= =?UTF-8?q?=20=E4=B8=8A=E9=99=90=E4=B8=8E=E4=B8=89=E5=A4=84=E5=85=A8?= =?UTF-8?q?=E8=A1=A8=E6=89=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一版是把 PG 的 DDL 逐句转写过来，留下四个实库可复现的缺陷。这一轮按 MySQL 自己的 写法重做基线（迁移未发布，直接改基线而非追加补丁，已与用户确认）。 - 自然键回到真列：live_location_key 那个 CONCAT(...CHAR(0)...) 生成列在 utf8mb4_0900_ai_ci 下分隔符权重为空、会被忽略，('proj','Xdev') 与 ('projX','dev') 拼成同一个键而误判重复。改成 (user_id, project_sync_id, agentred_fingerprint, live_location_flag) 四列唯一键，用「存活时为 1、否则 NULL」的标志列表达部分唯一索引 - 标识与凭据列一律 utf8mb4_bin：默认排序规则大小写不敏感，sync_id 'abc'/'ABC' 会互相 顶掉、且 WHERE sync_id=? 会取回另一行。表级 ENGINE/CHARSET/COLLATE 全部写死， 不再随 character_set_server 变化 - sync_avatars.content 由 text 改 mediumtext：text 上限 65535 字节，而 sync_svc.MaxAvatarBytes 是 4 MiB，超过 64KB 的头像直接 ER_DATA_TOO_LONG - users / device_flow_codes 的部分唯一索引改用标志列，键首列放业务列，使同一索引既做 约束又能服务 FindByEmail 与 user_code 轮询——两处原本都是全表扫，后者还是每 5 秒一次 的 device flow 热路径且含两条 UPDATE - device_code 按生成器实际产出收窄到 varchar(64)（randomBase32(32) 恒为 52 位）， 它是主键，InnoDB 会把它塞进每条二级索引 sync_repo.Save 不再用 ON DUPLICATE KEY UPDATE：MySQL 命中的是任意唯一键， sync_objects 上有两个，自然键冲突时它会静默改写另一个 sync_id 的那一行，本次上行的 对象从未落库而调用方拿到成功。改成「带版本条件的 UPDATE + 裸 INSERT」，用新增的 internal/pkg/dberr.IsDuplicateKey 按索引名区分 1062：撞身份键是版本竞败（吞掉）， 撞自然键是 R4b 兜底（上抛）。 其余： - 恢复被删的 migrations/migrations_test.go 并补 GET_LOCK 返回 NULL 的用例（原先 NULL 会变成一条与等锁无关的驱动扫描错误）；go test ./migrations/ 不再是 no test files - 去掉 testutils 里把实际 SQL 改写成 PG 方言的匹配层，全部期望改写为真实 MySQL 语句 - raw_profile 的默认值交回 schema（DEFAULT ('{}')），删掉与 user_svc 重复的仓储层兜底 - make test-e2e 显式跑 runner-test，否则 web/ 被冒烟轨道排除、它一次都不会跑 - 修掉机械替换留下的半成品注释与文档（ON DUPLICATE KEY (cols) DO NOTHING 之类不存在的 语法、「GET_LOCK 轮询而非阻塞的 GET_LOCK」这种同义对比、残留的 PG/Redis 字样） --- .github/workflows/ci.yml | 2 +- Makefile | 10 +- docs/architecture.md | 26 +++- .../specs/2026-08-07-multi-instance-safety.md | 4 +- .../2026-08-08-drop-device-capabilities.md | 2 +- docs/testing.md | 12 +- e2e/playwright.config.ts | 2 +- e2e/playwright.scratch.config.ts | 2 +- e2e/web/full-chain.spec.ts | 2 +- go.mod | 2 +- internal/model/entity/sync_entity/sync.go | 3 +- .../user_identity_entity/user_identity.go | 8 +- internal/pkg/dberr/dberr.go | 33 +++++ internal/pkg/dberr/dberr_test.go | 70 ++++++++++ .../device_flow_repo/device_flow_test.go | 12 +- internal/repository/device_repo/device.go | 7 +- .../device_token_repo/device_token_test.go | 20 +-- internal/repository/follow_repo/follow.go | 2 +- .../repository/follow_repo/follow_test.go | 8 +- internal/repository/sync_repo/object.go | 64 ++++++--- internal/repository/sync_repo/state.go | 10 +- internal/repository/sync_repo/sync_test.go | 117 +++++++++++++--- .../user_identity_repo/user_identity.go | 3 - .../user_identity_repo/user_identity_test.go | 36 ++++- internal/repository/user_repo/user_test.go | 6 +- internal/testutils/database.go | 23 +--- migrations/202605200001_users.go | 20 ++- migrations/202605200002_user_identities.go | 18 ++- migrations/202605200003_devices.go | 16 ++- migrations/202605200004_device_tokens.go | 17 ++- migrations/202605200005_device_flow_codes.go | 33 +++-- migrations/202608090001_workspace_sync.go | 60 ++++++--- migrations/202608100001_followed_sessions.go | 12 +- migrations/migrations.go | 23 +++- migrations/migrations_test.go | 126 ++++++++++++++++++ 35 files changed, 650 insertions(+), 161 deletions(-) create mode 100644 internal/pkg/dberr/dberr.go create mode 100644 internal/pkg/dberr/dberr_test.go create mode 100644 migrations/migrations_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04561c9e..d6a2c606 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: - name: make test run: make test - # 冒烟 e2e：桌面 + 移动两个 project，API 全 mock，不需要 PG/Redis。 + # 冒烟 e2e：桌面 + 移动两个 project，API 全 mock，不需要 MySQL/Redis。 e2e: name: e2e runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index bb316126..0a04ce52 100644 --- a/Makefile +++ b/Makefile @@ -37,12 +37,16 @@ test-backend: prepare-web-dist test-frontend: cd frontend && pnpm install --frozen-lockfile --silent && pnpm test -# 冒烟 e2e（桌面 + 移动两个 project）。scratch 轨道见 e2e/README.md。 +# 冒烟 e2e（桌面 + 移动两个 project）+ runner 自身的单测。scratch 轨道见 e2e/README.md。 # 浏览器由 pnpm smoke 自己装，这里只补 CI 要的系统库（mac 上是空跑）。 +# +# runner-test 不开浏览器，只测 run-e2e-web.mjs 里那些纯函数（改写 server 配置、解析 +# DSN 与 Redis、诊断启动失败）。它住在 e2e/web/ 下，而 web/ 被冒烟轨道整个排除了， +# 所以必须在这里显式点名——否则它一次都不会跑，等于没有。 test-e2e: - cd e2e && pnpm install --frozen-lockfile --silent && pnpm exec playwright install-deps chromium && pnpm smoke + cd e2e && pnpm install --frozen-lockfile --silent && pnpm exec playwright install-deps chromium && pnpm runner-test && pnpm smoke -# web 全链路 e2e（真浏览器 + 真 server + 真 agentred，开发环境 PG/Redis）。 +# web 全链路 e2e（真浏览器 + 真 server + 真 agentred，开发环境 MySQL/Redis）。 # 不进 CI、不进 make test——按需运行，见 e2e/README.md「web 全链路」一节。 test-e2e-web: cd e2e && pnpm exec playwright install chromium && pnpm web diff --git a/docs/architecture.md b/docs/architecture.md index e0519b49..c65a2ec2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -194,9 +194,22 @@ the transaction and a "deny" committed in that gap would otherwise still hand th token. A write with no conditional `UPDATE` to hang the decision on needs the database to arbitrate -some other way. `device_repo.Upsert` is a single `INSERT … ON DUPLICATE KEY ("user_id", -"fingerprint") DO UPDATE`, not a find-then-create, so two exchanges for the same device -converge on one row instead of racing to a `uk_devices_user_fingerprint` duplicate-key 500. +some other way. `device_repo.Upsert` writes with `INSERT … ON DUPLICATE KEY UPDATE` and then +reads the settled row back inside the same transaction (MySQL has no `RETURNING`), rather +than a find-then-create, so two exchanges for the same device converge on one row instead of +racing to a `uk_devices_user_fingerprint` duplicate-key 500. + +**`ON DUPLICATE KEY UPDATE` only arbitrates when the table has exactly one unique key.** +MySQL fires it on whichever unique key the row collided with, and does not tell you which — +`clause.OnConflict{Columns: …}` is decorative in the MySQL dialect. `devices`, +`sync_account_seqs`, `sync_device_states`, `sync_avatars` and `followed_sessions` each have a +single unique key, so the clause means what it reads like. `sync_objects` has two +(`uk_sync_objects_identity` and `uk_sync_objects_location`), and there the clause would +quietly rewrite *another account row's* content under its own `sync_id`. `sync_repo.Save` +therefore splits into a version-guarded `UPDATE` plus a plain `INSERT`, and discriminates the +resulting `1062` by index name via `internal/pkg/dberr.IsDuplicateKey` — an identity +collision is a lost version race and is swallowed, a location collision is the R4b backstop +and must surface. Before adding an upsert, count the table's unique keys. Inside a transaction, put the conditional `UPDATE` **first**, before any write that depends on winning: `ExchangeToken` marks the flow consumed before it touches `devices`, and @@ -219,8 +232,11 @@ in-process guard. `migrations/migrations.go`'s `RunMigrations` takes a MySQL nam lock (`withMigrationLock`) on a connection obtained via `sqlDB.Conn(ctx)` before running gormigrate, because named locks are session-scoped and a `*gorm.DB` call can otherwise land on a different pooled connection than the one that acquired the lock. It polls -`GET_LOCK` rather than blocking, up to a 120s budget, so a replica that can't -get the lock fails loudly instead of hanging past its startup probe. +`GET_LOCK(name, 0)` — the zero-timeout form, which returns immediately — rather than letting +`GET_LOCK(name, )` block, up to a 120s budget, so a replica that can't get the lock +fails loudly instead of hanging past its startup probe. `GET_LOCK` has three outcomes, not +two: `1` acquired, `0` held by someone else, and `NULL` when an error occurred; only `1` +counts as acquired, and the other two keep polling until the budget runs out. ## How to add an X diff --git a/docs/specs/2026-08-07-multi-instance-safety.md b/docs/specs/2026-08-07-multi-instance-safety.md index 1d632ecd..a7b89a5c 100644 --- a/docs/specs/2026-08-07-multi-instance-safety.md +++ b/docs/specs/2026-08-07-multi-instance-safety.md @@ -45,8 +45,8 @@ |---|---|---| | 1 | 8 项修复放同一轮交付 | 用户决定。Rejected: 拆成「基础设施加锁」与「Device Flow 状态机原子化」两轮——后者会改变并发竞败方的可观察错误、按 `docs/testing.md` 需走真 MySQL 的 scratch 验证，风险类别与前者不同，分开评审和回滚的粒度更细。用户已知该取舍并选择一轮完成 | | 2 | 多实例约束只写文档，不加机械守卫 | 用户决定。 | -| 3 | 迁移用 MySQL named lock 串行化，锁持有在一条专用连接上 | advisory lock 是**会话级**的，而 gorm 从连接池取连接，直接 `gdb.Exec("GET_LOCK")` 可能在 A 连接上加锁、在 B 连接上跑迁移，锁会随 A 归还池中而失去意义。因此从 `sqlDB.Conn(ctx)` 取一条固定连接持锁，迁移本身照常走连接池——advisory lock 不绑定数据，这样是正确的。进程崩溃时连接断开，MySQL 自动释放。Rejected: helm `pre-upgrade` Job 单独跑迁移——能解决问题，但 `helm upgrade` 之外的启动路径（本地 `make dev`、`docker-compose`、手工 `kubectl run`）就不再有保护，问题从代码里搬到了部署方式里 | -| 4 | 用 `GET_LOCK` 轮询而非阻塞的 `GET_LOCK` | 阻塞版没有上界，前面的副本卡住会让后面的副本静默挂起到被探针杀掉，日志里什么都看不到。轮询版有明确的等待预算，超时就返回错误、由 `main.go` 打日志退出，CrashLoop 是可见且自愈的。副作用是可以用 sqlmock 断言「拿不到锁时会重试、拿到后才跑迁移、结束后解锁」这个序列 | +| 3 | 迁移用 MySQL named lock 串行化，锁持有在一条专用连接上 | named lock 是**会话级**的，而 gorm 从连接池取连接，直接 `gdb.Exec("GET_LOCK")` 可能在 A 连接上加锁、在 B 连接上跑迁移，锁会随 A 归还池中而失去意义。因此从 `sqlDB.Conn(ctx)` 取一条固定连接持锁，迁移本身照常走连接池——named lock 不绑定数据，这样是正确的。进程崩溃时连接断开，MySQL 自动释放。Rejected: helm `pre-upgrade` Job 单独跑迁移——能解决问题，但 `helm upgrade` 之外的启动路径（本地 `make dev`、`docker-compose`、手工 `kubectl run`）就不再有保护，问题从代码里搬到了部署方式里 | +| 4 | 用 `GET_LOCK(name, 0)` 轮询，而非让 `GET_LOCK(name, )` 自己阻塞 | 阻塞版的上界由那个 timeout 参数决定，一旦取大（或取 -1 永久等待），前面的副本卡住会让后面的副本静默挂起到被探针杀掉，日志里什么都看不到。零超时 + 自己轮询有明确的等待预算，超时就返回错误、由 `main.go` 打日志退出，CrashLoop 是可见且自愈的。副作用是可以用 sqlmock 断言「拿不到锁时会重试、拿到后才跑迁移、结束后解锁」这个序列。注意 `GET_LOCK` 有三种返回：`1` 拿到、`0` 被别人持有、`NULL` 发生错误——只有 `1` 算拿到 | | 5 | 迁移等待预算 120s，同时把 chart 的 `startupProbe.failureThreshold` 从 30 提到 60 | 现值 `periodSeconds: 5 × failureThreshold: 30` = 150s 总预算，而等待锁 120s 之后还要真正跑迁移，很容易在迁移中途被探针杀掉；`UseTransaction: false` 下被杀在中途会留下半应用的迁移。提到 60（300s）让「等锁 + 迁移」有富余。Rejected: 缩短等待预算——迁移本身可能就要几十秒，等待预算小于它没有意义 | | 6 | 定时任务用 Redis 锁「占用当期」，**不主动解锁**，靠 TTL 自然过期，TTL 取略小于 cron 周期 | cago 的 `pkg/sync` locker 的 `UnlockKey` 是无条件 `DEL`、不校验持有者（`sync/redis.go`），任务一旦跑超 TTL，别的副本已经拿到锁，而先前那个副本的 Unlock 会把**别人的**锁删掉。改成「只 TryLock、不 Unlock」就完全绕开了这个问题：锁的语义正好是「本周期已被某个副本认领」，也正是 cron 需要的语义。TTL 取 `*/5` → 4m、`0 * * * *` → 50m。Rejected: 自己写带 owner token + Lua CAS 删除的锁——正确但等于在仓库里放第二套锁实现，而 cron 场景不需要提前释放；Rejected: 直接用 cago 的 Lock/Unlock 配对——即上述删错锁的缺陷 | | 7 | 拿不到锁时任务返回 `nil` 而非错误 | 「另一个副本正在跑」是预期内的正常路径，不是故障。返回错误会让 cago 的 crontab 包装器（`crontab.go:37`）在每个没抢到锁的副本上打一条 `cron error`，N-1 份噪音会把真正的失败淹掉 | diff --git a/docs/specs/2026-08-08-drop-device-capabilities.md b/docs/specs/2026-08-08-drop-device-capabilities.md index 4a8dfe43..891a89c0 100644 --- a/docs/specs/2026-08-08-drop-device-capabilities.md +++ b/docs/specs/2026-08-08-drop-device-capabilities.md @@ -104,7 +104,7 @@ | `device_svc` 单测 | 授权入参不再携带能力；pending 应答与设备列表项不含能力；签发的 claims 不含 caps | `internal/service/device_svc/device_test.go` 现有 sqlmock + mockgen 结构 | | `device_repo` 单测 | upsert 的赋值列不含 `capabilities`，冲突键与其余赋值列不变（R9） | `internal/repository/device_repo/device_test.go:31` 现有 SQL 断言 | | `jwt` 单测 | 签发 / 验签往返不含 caps；**一枚带 caps 的旧令牌仍验签通过**（R7 的兼容承诺） | `internal/pkg/jwt/jwt_test.go:24` | -| 迁移单测 | 新迁移排在 `migrationList()` 末尾，Migrate 与 Rollback 各自执行预期 SQL | `migrations/migrations_test.go` 现有 sqlmock 结构 | +| 迁移单测 | 新迁移排在 `migrationList()` 末尾，Migrate 与 Rollback 各自执行预期 SQL | `migrations/migrations_test.go` 现有 sqlmock 结构（只覆盖锁包装器，DDL 按 docs/testing.md 走实库人工验证） | | 控制层（muxtest） | authorize 请求体里带着 `capabilities` 时照常成功且被忽略（R4） | `internal/controller/device_ctr/device_test.go` | | 前端 vitest | 确认屏出现无条件的完整权限说明且与 kind 无关（R1）；页面上不再有任何能力摘要（R2）；倒计时 / 拒绝 / 允许 / 无 dialog 的既有用例原样通过（R3） | `frontend/src/__tests__/device-approval-risk.test.tsx` | | locale 对等 | 新增与删除的键在两个 locale 一致 | `frontend/src/i18n/__tests__/locale-parity.test.ts` | diff --git a/docs/testing.md b/docs/testing.md index b5ebde43..9b2fd5fb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -25,6 +25,13 @@ Repository tests are the rule people break first. sqlmock keeps them fast and he a real database makes them order-dependent and slow, and they start failing for reasons that have nothing to do with the code. +**Write expectations in the dialect the driver actually speaks** — backtick-quoted +identifiers and `?` placeholders. `testutils.Database` deliberately has no dialect +translation layer: rewriting the emitted SQL into some other dialect before matching means +the test pins a string the database will never receive, and the next reader concludes the +service talks to a different engine than it does. If an expectation looks wrong against +MySQL, the expectation is wrong. + There is no cross-layer tier. A test that stands up its own `gin.New()` and hand-writes the `code`/`msg`/`data` envelope is not testing the wiring — it is testing a second implementation of it, one that stays green while the real `internal/api/router.go` breaks. @@ -84,9 +91,10 @@ Write that check up under `e2e/scratch/` per [verification.md](verification.md) migrations the evidence is the table list, not a screenshot. **What is untested is the DDL, not the runner.** `migrations/migrations_test.go` does use -sqlmock, on the advisory-lock wrapper `withMigrationLock` that serialises concurrently +sqlmock, on the named-lock wrapper `withMigrationLock` that serialises concurrently starting replicas — it asserts the `GET_LOCK` retry, that the migration func -only runs once the lock is held, and that `RELEASE_LOCK` follows. That is a +only runs once the lock is held, that `RELEASE_LOCK` follows, and that a `NULL` from +`GET_LOCK` counts as *not* acquired. That is a statement-sequence assertion, so it stays hermetic; it says nothing about whether any migration in `migrationList()` is valid SQL. diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 91d7cf19..a035a15d 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -40,7 +40,7 @@ export default defineConfig({ ], // 前端 dev server 就够跑通 SPA 外壳（路由 / 主题 / i18n / 响应式）。 - // 需要真实后端的流程（设备流全链路）要 PG + Redis，属于 scratch 轨道， + // 需要真实后端的流程（设备流全链路）要 MySQL + Redis，属于 scratch 轨道， // 见 README.md 的「需要真后端时」。 webServer: { command: FRONTEND_DEV_COMMAND, diff --git a/e2e/playwright.scratch.config.ts b/e2e/playwright.scratch.config.ts index 89008ad5..4bed1b6c 100644 --- a/e2e/playwright.scratch.config.ts +++ b/e2e/playwright.scratch.config.ts @@ -35,7 +35,7 @@ export default defineConfig({ ], // scratch 默认不自动起服务：一次性验证往往要连你手动拉起来的那套 - // （真 server + 它 configs/config.yaml 里指的那套 PG/Redis）。 + // （真 server + 它 configs/config.yaml 里指的那套 MySQL/Redis）。 // 带上 E2E_SCRATCH_AUTOSTART=1 就顺带起前端，/v1 由 vite proxy 转到 :8443。 webServer: process.env.E2E_SCRATCH_AUTOSTART ? { diff --git a/e2e/web/full-chain.spec.ts b/e2e/web/full-chain.spec.ts index 8d2825d6..a8f8d69d 100644 --- a/e2e/web/full-chain.spec.ts +++ b/e2e/web/full-chain.spec.ts @@ -1,7 +1,7 @@ /** * WEB FULL-CHAIN e2e —— 测试接缝 10:一次真实运行走通全链路。 * - * 真浏览器 + 本机真 agentre-server(开发环境 PG/Redis)+ 真 agentred(fake + * 真浏览器 + 本机真 agentre-server(开发环境 MySQL/Redis)+ 真 agentred(fake * runtime,字节稳定回复)。run-e2e-web.mjs 已经: * - 播种一个一次性账号(PG)+ agentred 设备 + Redis 浏览器 session; * - 写好已认领的 agentred state.json 并跑起 daemon(连上中继); diff --git a/go.mod b/go.mod index 9ad65d35..a971c86b 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/cago-frame/cago v0.0.0-20260423074423-9a4902409f48 github.com/gin-gonic/gin v1.10.0 github.com/go-gormigrate/gormigrate/v2 v2.1.2 + github.com/go-sql-driver/mysql v1.8.1 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/gorilla/websocket v1.5.3 github.com/oklog/ulid/v2 v2.1.1 @@ -50,7 +51,6 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.23.0 // indirect - github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/internal/model/entity/sync_entity/sync.go b/internal/model/entity/sync_entity/sync.go index cadb6d42..2ceb7b54 100644 --- a/internal/model/entity/sync_entity/sync.go +++ b/internal/model/entity/sync_entity/sync.go @@ -60,7 +60,8 @@ func (*SyncObject) TableName() string { return "sync_objects" } func (o *SyncObject) IsDeleted() bool { return o != nil && o.DeletedAt > 0 } // 账号级单调递增的版本序列住在 sync_account_seqs 表里，这里**刻意没有**对应的 -// entity：分配版本必须用 `INSERT … ON DUPLICATE KEY UPDATE` +// entity：分配版本必须由数据库一次做完（`INSERT … ON DUPLICATE KEY UPDATE` 配 +// `LAST_INSERT_ID`） // （见 sync_repo.NextVersion），一个 gorm 结构体只会引来先读后写那种用法，而先读 // 后写在多副本并发上行时会把同一个版本号发给两次上行，R4 的「较大者胜」立刻失去 // 可比性。表结构以迁移里的 DDL 为准。 diff --git a/internal/model/entity/user_identity_entity/user_identity.go b/internal/model/entity/user_identity_entity/user_identity.go index f1bfcee6..2154b250 100644 --- a/internal/model/entity/user_identity_entity/user_identity.go +++ b/internal/model/entity/user_identity_entity/user_identity.go @@ -10,9 +10,11 @@ type UserIdentity struct { ProviderUID string `gorm:"column:provider_uid;type:text;not null"` ProviderLogin string `gorm:"column:provider_login;type:text;not null;default:''"` Email string `gorm:"column:email;type:text;not null"` - RawProfile []byte `gorm:"column:raw_profile;type:json;not null"` - Createtime int64 `gorm:"column:createtime;type:bigint;not null;default:0"` - Updatetime int64 `gorm:"column:updatetime;type:bigint;not null;default:0"` + // RawProfile 带 default：空值时 gorm 会把这一列整个从 INSERT 里省掉，交给 + // schema 的 DEFAULT ('{}')。「没有 profile 就存 {}」因此只有一处实现。 + RawProfile []byte `gorm:"column:raw_profile;type:json;not null;default:'{}'"` + Createtime int64 `gorm:"column:createtime;type:bigint;not null;default:0"` + Updatetime int64 `gorm:"column:updatetime;type:bigint;not null;default:0"` } func (*UserIdentity) TableName() string { return "user_identities" } diff --git a/internal/pkg/dberr/dberr.go b/internal/pkg/dberr/dberr.go new file mode 100644 index 00000000..57a2f153 --- /dev/null +++ b/internal/pkg/dberr/dberr.go @@ -0,0 +1,33 @@ +// Package dberr 把驱动层的数据库错误翻译成仓储层能分支的判断。 +// +// 只放「读错误」的函数，不碰连接、不碰 gorm，因此 repository 可以导入它而不违反 +// internal/pkg 的依赖方向。 +package dberr + +import ( + "errors" + "strings" + + "github.com/go-sql-driver/mysql" +) + +// errDupEntry 是 MySQL 的 ER_DUP_ENTRY。 +const errDupEntry = 1062 + +// IsDuplicateKey 判断 err 是不是 index 这个唯一键上的重复键错误。 +// +// 必须收敛到具体索引名，不能只判「是不是 1062」：一张表上有多个唯一键时，撞哪一个 +// 决定了业务该吞掉还是该报错。sync_objects 就是这样——撞身份键是版本竞败（吞掉）， +// 撞自然键是 R4b 兜底（必须响）。把两者混在一起等于把该抛的错吞掉。 +// +// 索引名只出现在错误文本里（`Duplicate entry 'x' for key 'tbl.idx'`），这是 MySQL +// 唯一暴露它的地方。MySQL 8 起带表名前缀，5.7 不带，两种形态都认。用后缀而不是 +// 包含匹配：uk_x 是 uk_x_v2 的前缀，包含匹配会把后者的冲突认成前者的。 +func IsDuplicateKey(err error, index string) bool { + var myErr *mysql.MySQLError + if !errors.As(err, &myErr) || myErr.Number != errDupEntry { + return false + } + return strings.HasSuffix(myErr.Message, "'"+index+"'") || + strings.HasSuffix(myErr.Message, "."+index+"'") +} diff --git a/internal/pkg/dberr/dberr_test.go b/internal/pkg/dberr/dberr_test.go new file mode 100644 index 00000000..3f52d205 --- /dev/null +++ b/internal/pkg/dberr/dberr_test.go @@ -0,0 +1,70 @@ +package dberr + +import ( + "errors" + "fmt" + "testing" + + "github.com/go-sql-driver/mysql" + "github.com/stretchr/testify/assert" +) + +// 一张表上有多个唯一键时，「撞了哪一个」决定业务该吞掉还是该报错，所以判断必须 +// 按索引名收敛到一个键上，不能只看「是不是 1062」。 +func TestIsDuplicateKey_MatchesOnlyTheNamedIndex(t *testing.T) { + err := &mysql.MySQLError{ + Number: 1062, + Message: "Duplicate entry '7-s1' for key 'sync_objects.uk_sync_objects_identity'", + } + + assert.True(t, IsDuplicateKey(err, "uk_sync_objects_identity")) + assert.False(t, IsDuplicateKey(err, "uk_sync_objects_location"), + "另一个唯一键上的冲突必须报错，不能被当成身份键冲突吞掉") +} + +// MySQL 8 起错误文本里的索引名带表名前缀，5.7 不带。两种都要认，否则换一个 +// 小版本就会静默失配——而失配的后果是把该抛的错吞掉。 +func TestIsDuplicateKey_AcceptsBothMessageForms(t *testing.T) { + qualified := &mysql.MySQLError{ + Number: 1062, + Message: "Duplicate entry '7-s1' for key 'sync_objects.uk_sync_objects_identity'", + } + bare := &mysql.MySQLError{ + Number: 1062, + Message: "Duplicate entry '7-s1' for key 'uk_sync_objects_identity'", + } + + assert.True(t, IsDuplicateKey(qualified, "uk_sync_objects_identity")) + assert.True(t, IsDuplicateKey(bare, "uk_sync_objects_identity")) +} + +// 索引名不能用「包含」去匹配：uk_x 是 uk_x_location 的前缀，用 Contains 会把 +// 后者的冲突认成前者的。 +func TestIsDuplicateKey_DoesNotMatchAnIndexNamePrefix(t *testing.T) { + err := &mysql.MySQLError{ + Number: 1062, + Message: "Duplicate entry '7-s1' for key 'sync_objects.uk_sync_objects_identity_v2'", + } + + assert.False(t, IsDuplicateKey(err, "uk_sync_objects_identity")) +} + +// 别的错误号（以及包装过的、nil 的）都不是重复键。 +func TestIsDuplicateKey_RejectsEverythingElse(t *testing.T) { + assert.False(t, IsDuplicateKey(nil, "uk_x")) + assert.False(t, IsDuplicateKey(errors.New("boom"), "uk_x")) + assert.False(t, IsDuplicateKey(&mysql.MySQLError{ + Number: 1406, + Message: "Data too long for column 'content' at row 1", + }, "uk_x")) +} + +// 错误经常被 fmt.Errorf("%w") 包过一层再往上传，errors.As 必须能穿透。 +func TestIsDuplicateKey_UnwrapsWrappedErrors(t *testing.T) { + wrapped := fmt.Errorf("save sync object: %w", &mysql.MySQLError{ + Number: 1062, + Message: "Duplicate entry '7-s1' for key 'sync_objects.uk_sync_objects_identity'", + }) + + assert.True(t, IsDuplicateKey(wrapped, "uk_sync_objects_identity")) +} diff --git a/internal/repository/device_flow_repo/device_flow_test.go b/internal/repository/device_flow_repo/device_flow_test.go index c04fda56..d45a213f 100644 --- a/internal/repository/device_flow_repo/device_flow_test.go +++ b/internal/repository/device_flow_repo/device_flow_test.go @@ -15,7 +15,7 @@ func TestApprove(t *testing.T) { r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( - `UPDATE "device_flow_codes" SET "approved_at"=$1,"authorized_user_id"=$2 WHERE user_code=$3 AND consumed_at=0 AND denied_at=0 AND expires_at > $4`, + "UPDATE `device_flow_codes` SET `approved_at`=?,`authorized_user_id`=? WHERE user_code=? AND consumed_at=0 AND denied_at=0 AND expires_at > ?", )).WithArgs(int64(1000), int64(99), "A4F-7Q2", int64(1000)). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectCommit() @@ -32,7 +32,7 @@ func TestApprove_ReturnsZeroRowsWhenNothingMatched(t *testing.T) { r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( - `UPDATE "device_flow_codes" SET "approved_at"=$1,"authorized_user_id"=$2 WHERE user_code=$3 AND consumed_at=0 AND denied_at=0 AND expires_at > $4`, + "UPDATE `device_flow_codes` SET `approved_at`=?,`authorized_user_id`=? WHERE user_code=? AND consumed_at=0 AND denied_at=0 AND expires_at > ?", )).WithArgs(int64(1000), int64(99), "A4F-7Q2", int64(1000)). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() @@ -48,7 +48,7 @@ func TestDeny(t *testing.T) { r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( - `UPDATE "device_flow_codes" SET "denied_at"=$1 WHERE user_code=$2 AND consumed_at=0 AND denied_at=0`, + "UPDATE `device_flow_codes` SET `denied_at`=? WHERE user_code=? AND consumed_at=0 AND denied_at=0", )).WithArgs(int64(1000), "A4F-7Q2"). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() @@ -67,7 +67,7 @@ func TestMarkConsumed_RequiresUnsettledRow(t *testing.T) { r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( - `UPDATE "device_flow_codes" SET "consumed_at"=$1 WHERE device_code=$2 AND consumed_at=0 AND denied_at=0`, + "UPDATE `device_flow_codes` SET `consumed_at`=? WHERE device_code=? AND consumed_at=0 AND denied_at=0", )).WithArgs(int64(1000), "dc-x"). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectCommit() @@ -84,7 +84,7 @@ func TestMarkConsumed_ReturnsZeroRowsWhenAlreadySettled(t *testing.T) { r := NewDeviceFlow() mock.ExpectBegin() mock.ExpectExec(regexp.QuoteMeta( - `UPDATE "device_flow_codes" SET "consumed_at"=$1 WHERE device_code=$2 AND consumed_at=0 AND denied_at=0`, + "UPDATE `device_flow_codes` SET `consumed_at`=? WHERE device_code=? AND consumed_at=0 AND denied_at=0", )).WithArgs(int64(1000), "dc-x"). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() @@ -98,7 +98,7 @@ func TestMarkConsumed_ReturnsZeroRowsWhenAlreadySettled(t *testing.T) { func TestFindByDeviceCode_Found(t *testing.T) { ctx, _, mock := hubtest.Database(t) r := NewDeviceFlow() - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "device_flow_codes" WHERE device_code=$1 ORDER BY "device_flow_codes"."device_code" LIMIT $2`)). + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `device_flow_codes` WHERE device_code=? ORDER BY `device_flow_codes`.`device_code` LIMIT ?")). WithArgs("dc-x", 1). WillReturnRows(sqlmock.NewRows([]string{"device_code", "user_code"}).AddRow("dc-x", "A4F-7Q2")) got, err := r.FindByDeviceCode(ctx, "dc-x") diff --git a/internal/repository/device_repo/device.go b/internal/repository/device_repo/device.go index 3f2b2a88..2b214b3c 100644 --- a/internal/repository/device_repo/device.go +++ b/internal/repository/device_repo/device.go @@ -57,8 +57,11 @@ func (r *repo) FindByFingerprint(ctx context.Context, userID int64, fp string) ( } // Upsert 按 (user_id, fingerprint) 落库：走 uk_devices_user_fingerprint 的 -// ON DUPLICATE KEY UPDATE 由数据库原子裁决。MySQL 没有通用的 -// INSERT ... transaction read-back，所以写入后在同一事务内读回最终行填充 d。 +// ON DUPLICATE KEY UPDATE 由数据库原子裁决。devices 上只有这一个唯一键，所以这条 +// 语句命中的必然是它——多唯一键的表不能这么写（见 sync_repo.Save）。 +// +// MySQL 没有 RETURNING，所以写入后在同一事务内读回最终行来填充 d：命中已有设备时 +// 拿到的是它原来的 id 与 createtime。 // // 不写成「先按 (user_id, fingerprint) 查、再 Save/Create」：那是先查后写，两个已授权的 // device_code 共用同一 (user_id, fingerprint) 并发换取时会双双查空、双双 INSERT， diff --git a/internal/repository/device_token_repo/device_token_test.go b/internal/repository/device_token_repo/device_token_test.go index bd3feb59..3ef40f78 100644 --- a/internal/repository/device_token_repo/device_token_test.go +++ b/internal/repository/device_token_repo/device_token_test.go @@ -16,7 +16,7 @@ func TestRevokeChain(t *testing.T) { r := NewDeviceToken() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`UPDATE "device_tokens" SET "revoked_at"=$1 WHERE device_id=$2 AND revoked_at=0`)). + mock.ExpectExec(regexp.QuoteMeta("UPDATE `device_tokens` SET `revoked_at`=? WHERE device_id=? AND revoked_at=0")). WithArgs(int64(1700000000000), int64(42)). WillReturnResult(sqlmock.NewResult(0, 3)) mock.ExpectCommit() @@ -30,7 +30,7 @@ func TestRevoke_RequiresUnrevokedRow(t *testing.T) { r := NewDeviceToken() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`UPDATE "device_tokens" SET "revoked_at"=$1 WHERE id=$2 AND revoked_at=0`)). + mock.ExpectExec(regexp.QuoteMeta("UPDATE `device_tokens` SET `revoked_at`=? WHERE id=? AND revoked_at=0")). WithArgs(int64(1700000000000), int64(11)). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectCommit() @@ -47,7 +47,7 @@ func TestRevoke_ReturnsZeroRowsWhenAlreadyRevoked(t *testing.T) { r := NewDeviceToken() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`UPDATE "device_tokens" SET "revoked_at"=$1 WHERE id=$2 AND revoked_at=0`)). + mock.ExpectExec(regexp.QuoteMeta("UPDATE `device_tokens` SET `revoked_at`=? WHERE id=? AND revoked_at=0")). WithArgs(int64(1700000000000), int64(11)). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() @@ -61,7 +61,7 @@ func TestRevoke_ReturnsZeroRowsWhenAlreadyRevoked(t *testing.T) { func TestFindByHash_Found(t *testing.T) { ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "device_tokens" WHERE refresh_token_hash=$1 ORDER BY "device_tokens"."id" LIMIT $2`)). + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `device_tokens` WHERE refresh_token_hash=? ORDER BY `device_tokens`.`id` LIMIT ?")). WithArgs("h1", 1). WillReturnRows(sqlmock.NewRows([]string{"id", "device_id"}).AddRow(int64(11), int64(42))) got, err := r.FindByHash(ctx, "h1") @@ -78,7 +78,7 @@ func TestCreate(t *testing.T) { // 十个 AnyArg 的期望连列名都不看，删掉 AccessJTI 字段照样绿，所以这里把列名和 // 那一列的值都钉死。 mock.ExpectExec(regexp.QuoteMeta( - `INSERT INTO "device_tokens" ("device_id","refresh_token_hash","access_jti"`)). + "INSERT INTO `device_tokens` (`device_id`,`refresh_token_hash`,`access_jti`")). WithArgs(int64(42), "h", "jti-1", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). @@ -92,7 +92,7 @@ func TestCreate(t *testing.T) { func TestListAccessJTIByDevice(t *testing.T) { ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() - mock.ExpectQuery(regexp.QuoteMeta(`SELECT "access_jti" FROM "device_tokens" WHERE device_id=$1 AND access_jti != ''`)). + mock.ExpectQuery(regexp.QuoteMeta("SELECT `access_jti` FROM `device_tokens` WHERE device_id=? AND access_jti != ''")). WithArgs(int64(42)). WillReturnRows(sqlmock.NewRows([]string{"access_jti"}).AddRow("jti-aaa").AddRow("jti-bbb")) got, err := r.ListAccessJTIByDevice(ctx, 42) @@ -105,7 +105,7 @@ func TestListRevokedJTIByUser(t *testing.T) { ctx, _, mock := hubtest.Database(t) r := NewDeviceToken() mock.ExpectQuery(regexp.QuoteMeta( - `SELECT "device_tokens"."access_jti" FROM "device_tokens" JOIN devices ON devices.id = device_tokens.device_id WHERE devices.user_id = $1 AND device_tokens.revoked_at != 0 AND device_tokens.access_jti != '' AND device_tokens.createtime >= $2`)). + "SELECT `device_tokens`.`access_jti` FROM `device_tokens` JOIN devices ON devices.id = device_tokens.device_id WHERE devices.user_id = ? AND device_tokens.revoked_at != 0 AND device_tokens.access_jti != '' AND device_tokens.createtime >= ?")). WithArgs(int64(7), int64(1000)). WillReturnRows(sqlmock.NewRows([]string{"access_jti"}).AddRow("jti-revoked-1").AddRow("jti-revoked-2")) got, err := r.ListRevokedJTIByUser(ctx, 7, 1000) @@ -121,7 +121,7 @@ func TestListRevokedJTIByUser_ReflectsRevokeImmediately(t *testing.T) { r := NewDeviceToken() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`UPDATE "device_tokens" SET "revoked_at"=$1 WHERE id=$2 AND revoked_at=0`)). + mock.ExpectExec(regexp.QuoteMeta("UPDATE `device_tokens` SET `revoked_at`=? WHERE id=? AND revoked_at=0")). WithArgs(int64(2000), int64(11)). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectCommit() @@ -130,7 +130,7 @@ func TestListRevokedJTIByUser_ReflectsRevokeImmediately(t *testing.T) { assert.Equal(t, int64(1), n) mock.ExpectQuery(regexp.QuoteMeta( - `SELECT "device_tokens"."access_jti" FROM "device_tokens" JOIN devices ON devices.id = device_tokens.device_id WHERE devices.user_id = $1 AND device_tokens.revoked_at != 0 AND device_tokens.access_jti != '' AND device_tokens.createtime >= $2`)). + "SELECT `device_tokens`.`access_jti` FROM `device_tokens` JOIN devices ON devices.id = device_tokens.device_id WHERE devices.user_id = ? AND device_tokens.revoked_at != 0 AND device_tokens.access_jti != '' AND device_tokens.createtime >= ?")). WithArgs(int64(7), int64(1000)). WillReturnRows(sqlmock.NewRows([]string{"access_jti"}).AddRow("jti-aaa")) got, err := r.ListRevokedJTIByUser(ctx, 7, 1000) @@ -145,7 +145,7 @@ func TestListRevokedJTIByUser_ExcludesOutsideAccessTTLWindow(t *testing.T) { // windowStartMs 之前签发的行已被数据库端的 createtime >= ? 条件排除， // mock 只按预期 SQL 返回窗口内的一行——验证调用方传入的 windowStartMs 确实被当成查询条件。 mock.ExpectQuery(regexp.QuoteMeta( - `SELECT "device_tokens"."access_jti" FROM "device_tokens" JOIN devices ON devices.id = device_tokens.device_id WHERE devices.user_id = $1 AND device_tokens.revoked_at != 0 AND device_tokens.access_jti != '' AND device_tokens.createtime >= $2`)). + "SELECT `device_tokens`.`access_jti` FROM `device_tokens` JOIN devices ON devices.id = device_tokens.device_id WHERE devices.user_id = ? AND device_tokens.revoked_at != 0 AND device_tokens.access_jti != '' AND device_tokens.createtime >= ?")). WithArgs(int64(7), int64(5000)). WillReturnRows(sqlmock.NewRows([]string{"access_jti"}).AddRow("jti-in-window")) got, err := r.ListRevokedJTIByUser(ctx, 7, 5000) diff --git a/internal/repository/follow_repo/follow.go b/internal/repository/follow_repo/follow.go index a3a482c1..6be5e0e3 100644 --- a/internal/repository/follow_repo/follow.go +++ b/internal/repository/follow_repo/follow.go @@ -31,7 +31,7 @@ func NewFollow() FollowRepo { return &repo{} } type repo struct{} -// Follow 是一条语句的条件插入：ON DUPLICATE KEY ... DO NOTHING 由数据库原子裁决， +// Follow 是一条语句的条件插入：命中唯一索引时那条 INSERT 什么都不改，由数据库原子裁决， // 并发重复关注两边都成功、只落一行。先查再插会在两个副本同时首次关注时双双 // 走到 INSERT，竞败方撞唯一索引拿到一个约束错误。 func (r *repo) Follow(ctx context.Context, f *follow_entity.FollowedSession) error { diff --git a/internal/repository/follow_repo/follow_test.go b/internal/repository/follow_repo/follow_test.go index 53349204..09705bb9 100644 --- a/internal/repository/follow_repo/follow_test.go +++ b/internal/repository/follow_repo/follow_test.go @@ -12,8 +12,8 @@ import ( hubtest "agentre-server/internal/testutils" ) -// Follow 必须是一条语句：INSERT ... ON DUPLICATE KEY (user_id, device_fingerprint, -// session_id) DO NOTHING。重复关注（同账号、同一目标会话）命中唯一索引时是 +// Follow 必须是一条语句：一条 INSERT，命中 uk_followed_sessions_identity +// (user_id, device_fingerprint, session_id) 时什么都不改。重复关注（同账号、同一目标会话）是 // no-op，由数据库原子裁决：不新增行、也不重置首次关注时间——R12「关注幂等」在 // 数据层的落点。这里用 0 行受影响的结果模拟「已关注」的那次重复请求。 func TestFollow_SingleStatementOnConflictDoNothing(t *testing.T) { @@ -43,7 +43,7 @@ func TestUnfollow_DeleteIsIdempotent(t *testing.T) { r := NewFollow() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`DELETE FROM "followed_sessions"`)). + mock.ExpectExec(regexp.QuoteMeta("DELETE FROM `followed_sessions`")). WithArgs(int64(7), "fp-daemon-1", "sess-9"). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() @@ -66,7 +66,7 @@ func TestListByUser_AccountScoped(t *testing.T) { // 排序也钉在 SQL 上：sqlmock 按给定顺序回行，光比第一行的内容，把 ORDER BY // 整句删掉这个用例照样绿。「最近关注的排在前面」是 R13 列表的顺序承诺。 mock.ExpectQuery(regexp.QuoteMeta( - `FROM "followed_sessions" WHERE user_id=$1 ORDER BY followed_at DESC, id DESC`, + "FROM `followed_sessions` WHERE user_id=? ORDER BY followed_at DESC, id DESC", )).WithArgs(int64(7)).WillReturnRows(rows) out, err := r.ListByUser(ctx, 7) diff --git a/internal/repository/sync_repo/object.go b/internal/repository/sync_repo/object.go index fbc8fe1d..b2887390 100644 --- a/internal/repository/sync_repo/object.go +++ b/internal/repository/sync_repo/object.go @@ -5,10 +5,9 @@ import ( "context" "github.com/cago-frame/cago/database/db" - "gorm.io/gorm" - "gorm.io/gorm/clause" "agentre-server/internal/model/entity/sync_entity" + "agentre-server/internal/pkg/dberr" ) //go:generate mockgen -source object.go -destination mock_sync_repo/mock_object.go @@ -76,23 +75,54 @@ func (r *objectRepo) FindLocationByNaturalKey( return ret, nil } -// Save 是一条语句的条件 upsert：命中（user_id, sync_id）时只有版本号更大的那次 -// 写入才覆盖。多副本并发上行同一行时由数据库裁决，先查后写会让落后的那次把更新 -// 的那一版盖掉。createtime 不在赋值列里：命中已有行时保留它首次落地的时间。 +// Save 按（账号, 同步标识）落库，且只在版本号更大时才覆盖已有行。 +// +// **为什么不是一条 INSERT … ON DUPLICATE KEY UPDATE。** MySQL 的 ON DUPLICATE KEY +// 命中的是**任意**唯一键，而 sync_objects 上有两个：uk_sync_objects_identity 和 +// uk_sync_objects_location。自然键被另一个 sync_id 占着时（R4b 竞态的兜底），那条 +// 语句不会报错，而是去 UPDATE 别人那一行——身份键留旧的、内容换成新的，本次上行的 +// sync_id 从来没落库，调用方却拿到成功。客户端于是永远重推同一个 sync_id，每次都 +// 把别人那行再覆盖一遍。gorm 的 clause.OnConflict{Columns: …} 在 MySQL 方言下只是 +// 装饰，收不住这件事；MySQL 官方文档同样建议多唯一键的表别用 ON DUPLICATE KEY。 +// +// 所以按 MySQL 的写法拆成两步，两步都由数据库裁决、都不是先读后写： +// +// 1. 一条带版本条件的 UPDATE。命中 = 覆盖成功；并发的两次上行由行锁串行，落后的 +// 那次条件不成立、改不动任何东西。命中时 version 必然与原值不同（条件就是 +// version `version`, VALUES(`" + column + "`), `" + column + "`)", - ) + updated := db.Ctx(ctx).Model(&sync_entity.SyncObject{}). + Where("user_id=? AND sync_id=? AND version 0 { + return nil + } + + err := db.Ctx(ctx).Create(obj).Error + if dberr.IsDuplicateKey(err, "uk_sync_objects_identity") { + return nil } - return db.Ctx(ctx).Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "user_id"}, {Name: "sync_id"}}, - DoUpdates: clause.Assignments(assignments), - }).Create(obj).Error + return err } func (r *objectRepo) Tombstone(ctx context.Context, id, version, nowMs int64) (int64, error) { diff --git a/internal/repository/sync_repo/state.go b/internal/repository/sync_repo/state.go index 2bc136dd..1c36ee3f 100644 --- a/internal/repository/sync_repo/state.go +++ b/internal/repository/sync_repo/state.go @@ -32,9 +32,13 @@ func NewSyncState() SyncStateRepo { return &stateRepo{} } type stateRepo struct{} -// NextVersion 必须是一条语句。先读后写在多副本并发上行时会双双读到同一个值、 -// 两次上行拿到同一个版本号，R4 的「较大者胜」立刻失去可比性；INSERT … ON -// CONFLICT DO UPDATE … transaction read-back 把递增与取值合成一次，由数据库的行锁串行化。 +// NextVersion 的递增与取值必须由数据库一次做完。先读后写在多副本并发上行时会双双读到 +// 同一个值、两次上行拿到同一个版本号，R4 的「较大者胜」立刻失去可比性。 +// +// MySQL 没有 RETURNING，这里用它自己的写法：INSERT … ON DUPLICATE KEY UPDATE 里把新值 +// 套进 LAST_INSERT_ID(expr)，该函数在设置的同时把值记在**连接**上，紧接着一条 +// SELECT LAST_INSERT_ID() 就能取回。递增由行锁串行化。外面那层事务不是为了原子性， +// 而是为了把两条语句钉在同一条连接上——LAST_INSERT_ID 是连接级的，走连接池会取到别人的值。 func (r *stateRepo) NextVersion(ctx context.Context, userID int64, n int64) (int64, error) { if n <= 0 { n = 1 diff --git a/internal/repository/sync_repo/sync_test.go b/internal/repository/sync_repo/sync_test.go index 3d9bfa14..7c2e322b 100644 --- a/internal/repository/sync_repo/sync_test.go +++ b/internal/repository/sync_repo/sync_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + mysqldriver "github.com/go-sql-driver/mysql" "github.com/stretchr/testify/assert" "agentre-server/internal/model/entity/sync_entity" @@ -53,7 +54,7 @@ func TestFindDeviceState_GivenNoRow_ThenNilNil(t *testing.T) { ctx, _, mock := hubtest.Database(t) r := NewSyncState() - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "sync_device_states"`)). + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `sync_device_states`")). WillReturnRows(sqlmock.NewRows([]string{"user_id", "device_id", "last_sync_at"})) st, err := r.FindDeviceState(ctx, 7, 2) @@ -77,14 +78,23 @@ func TestTouchDeviceState_GivenNoRow_ThenUpsertsInOneStatement(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } -// 落库要在版本号更大时才覆盖：两个副本并发写同一行时由数据库裁决， -// 落后的那次不能把更新的那一版盖掉。 -func TestSaveObject_GivenExistingRow_ThenUpsertOnlyWhenVersionIsGreater(t *testing.T) { +// 落库的第一步是一条带版本条件的 UPDATE：版本更大才覆盖，两个副本并发写同一行时 +// 由行锁裁决，落后的那次条件不成立、改不动任何东西。 +// +// 绑定值一并钉死：SET 里恰好 9 列、没有 createtime（命中已有行要保留它首次落地的 +// 时间），WHERE 里 version 作为条件再绑一次——少了它就退化成「后到的一定赢」。 +func TestSaveObject_GivenGreaterVersion_ThenConditionalUpdateWins(t *testing.T) { ctx, _, mock := hubtest.Database(t) r := NewSyncObject() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`ON DUPLICATE KEY UPDATE`)).WillReturnResult(sqlmock.NewResult(1, 2)) + mock.ExpectExec(regexp.QuoteMeta( + "UPDATE `sync_objects` SET `agentred_fingerprint`=?,`deleted_at`=?,`kind`=?,`payload`=?,"+ + "`project_sync_id`=?,`source_device_id`=?,`sync_updated_at`=?,`updatetime`=?,`version`=? "+ + "WHERE user_id=? AND sync_id=? AND version `version`, VALUES\\(`version`\\), `version`\\)"). - WillReturnResult(sqlmock.NewResult(1, 2)) + mock.ExpectExec(regexp.QuoteMeta("UPDATE `sync_objects` SET")). + WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() + mock.ExpectBegin() + mock.ExpectExec(regexp.QuoteMeta("INSERT INTO `sync_objects`")). + WillReturnError(&mysqldriver.MySQLError{ + Number: 1062, + Message: "Duplicate entry '7-p1' for key 'sync_objects.uk_sync_objects_identity'", + }) + mock.ExpectRollback() assert.NoError(t, r.Save(ctx, &sync_entity.SyncObject{ UserID: 7, Kind: sync_entity.KindProject, SyncID: "p1", Payload: `{}`, Version: 9, @@ -109,6 +150,50 @@ func TestSaveObject_GivenExistingRow_ThenConditionalWhereIsPresent(t *testing.T) assert.NoError(t, mock.ExpectationsWereMet()) } +// 撞上自然键是另一回事，必须响：自然键上另一个 sync_id 还活着，这是 R4b 的兜底。 +// 吞掉它意味着本次上行的对象从来没落库、调用方却拿到成功——客户端会永远重推同一个 +// sync_id，而库里那一行始终是别人的身份。这条断言就是为了挡住那种「静默成功」。 +func TestSaveObject_GivenLocationConflict_ThenErrorIsLoud(t *testing.T) { + ctx, _, mock := hubtest.Database(t) + r := NewSyncObject() + + locationConflict := &mysqldriver.MySQLError{ + Number: 1062, + Message: "Duplicate entry '7-proj-1-fp-a-1' for key 'sync_objects.uk_sync_objects_location'", + } + mock.ExpectBegin() + mock.ExpectExec(regexp.QuoteMeta("UPDATE `sync_objects` SET")). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectCommit() + mock.ExpectBegin() + mock.ExpectExec(regexp.QuoteMeta("INSERT INTO `sync_objects`")).WillReturnError(locationConflict) + mock.ExpectRollback() + + err := r.Save(ctx, &sync_entity.SyncObject{ + UserID: 7, Kind: sync_entity.KindProjectLocation, SyncID: "loc-B", + ProjectSyncID: "proj-1", AgentredFingerprint: "fp-a", Payload: `{}`, Version: 9, + }) + assert.ErrorIs(t, err, locationConflict) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +// 条件 UPDATE 自身的失败要如实上抛，不能被当成「没命中」而掉进 INSERT 分支—— +// 那会把一个数据库错误变成一次多余的写入尝试。 +func TestSaveObject_GivenUpdateFails_ThenErrorPropagatesWithoutInsert(t *testing.T) { + ctx, _, mock := hubtest.Database(t) + r := NewSyncObject() + + mock.ExpectBegin() + mock.ExpectExec(regexp.QuoteMeta("UPDATE `sync_objects` SET")).WillReturnError(assert.AnError) + mock.ExpectRollback() + + err := r.Save(ctx, &sync_entity.SyncObject{ + UserID: 7, Kind: sync_entity.KindProject, SyncID: "p1", Payload: `{}`, Version: 9, + }) + assert.ErrorIs(t, err, assert.AnError) + assert.NoError(t, mock.ExpectationsWereMet()) +} + // 自然键查重只看存活的那一行：墓碑不占（账号, 项目, 指纹），否则删掉再建就建不回来。 func TestFindLocationByNaturalKey_GivenTombstones_ThenOnlyLiveRowMatches(t *testing.T) { ctx, _, mock := hubtest.Database(t) @@ -130,7 +215,7 @@ func TestTombstone_GivenAlreadyTombstoned_ThenZeroRowsAffected(t *testing.T) { r := NewSyncObject() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`UPDATE "sync_objects" SET`)). + mock.ExpectExec(regexp.QuoteMeta("UPDATE `sync_objects` SET")). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectCommit() @@ -184,9 +269,9 @@ func TestReplaceSnapshot_GivenItems_ThenDeletesThenInsertsInOneTransaction(t *te r := NewSyncLocalPath() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`DELETE FROM "device_local_paths"`)). + mock.ExpectExec(regexp.QuoteMeta("DELETE FROM `device_local_paths`")). WillReturnResult(sqlmock.NewResult(0, 3)) - mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO "device_local_paths"`)). + mock.ExpectExec(regexp.QuoteMeta("INSERT INTO `device_local_paths`")). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectCommit() @@ -202,7 +287,7 @@ func TestReplaceSnapshot_GivenEmptySnapshot_ThenOnlyDeletes(t *testing.T) { r := NewSyncLocalPath() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`DELETE FROM "device_local_paths"`)). + mock.ExpectExec(regexp.QuoteMeta("DELETE FROM `device_local_paths`")). WillReturnResult(sqlmock.NewResult(0, 3)) mock.ExpectCommit() @@ -217,7 +302,7 @@ func TestDeleteByDevice_GivenDeviceID_ThenDeletesAllRowsForThatDevice(t *testing r := NewSyncLocalPath() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`DELETE FROM "device_local_paths" WHERE device_id=`)). + mock.ExpectExec(regexp.QuoteMeta("DELETE FROM `device_local_paths` WHERE device_id=")). WithArgs(int64(2)). WillReturnResult(sqlmock.NewResult(0, 3)) mock.ExpectCommit() @@ -232,7 +317,7 @@ func TestListByDevice_GivenDeviceID_ThenReturnsItsReportedRows(t *testing.T) { ctx, _, mock := hubtest.Database(t) r := NewSyncLocalPath() - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "device_local_paths" WHERE user_id=`)). + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `device_local_paths` WHERE user_id=")). WithArgs(int64(7), int64(2)). WillReturnRows(sqlmock.NewRows([]string{"user_id", "device_id", "project_sync_id", "path"}). AddRow(int64(7), int64(2), "p1", "/srv/p1")) @@ -268,7 +353,7 @@ func TestDeleteTombstonesBefore_GivenCutoff_ThenOnlyExpiredTombstonesAreDeleted( r := NewSyncObject() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`DELETE FROM "sync_objects" WHERE deleted_at>0 AND deleted_at<`)). + mock.ExpectExec(regexp.QuoteMeta("DELETE FROM `sync_objects` WHERE deleted_at>0 AND deleted_at<")). WithArgs(int64(1700)). WillReturnResult(sqlmock.NewResult(0, 3)) mock.ExpectCommit() @@ -302,7 +387,7 @@ func TestFindAvatar_GivenNoRow_ThenNilNil(t *testing.T) { ctx, _, mock := hubtest.Database(t) r := NewSyncAvatar() - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "sync_avatars"`)). + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `sync_avatars`")). WillReturnRows(sqlmock.NewRows([]string{"user_id", "content_hash"})) got, err := r.Find(ctx, 7, "h1") diff --git a/internal/repository/user_identity_repo/user_identity.go b/internal/repository/user_identity_repo/user_identity.go index 65fb8db5..63dd8389 100644 --- a/internal/repository/user_identity_repo/user_identity.go +++ b/internal/repository/user_identity_repo/user_identity.go @@ -25,9 +25,6 @@ func NewUserIdentity() UserIdentityRepo { return &repo{} } type repo struct{} func (r *repo) Create(ctx context.Context, e *user_identity_entity.UserIdentity) error { - if len(e.RawProfile) == 0 { - e.RawProfile = []byte("{}") - } return db.Ctx(ctx).Create(e).Error } diff --git a/internal/repository/user_identity_repo/user_identity_test.go b/internal/repository/user_identity_repo/user_identity_test.go index c62a8efe..19265752 100644 --- a/internal/repository/user_identity_repo/user_identity_test.go +++ b/internal/repository/user_identity_repo/user_identity_test.go @@ -16,7 +16,7 @@ func TestFindByProviderUID_Found(t *testing.T) { r := NewUserIdentity() mock.ExpectQuery(regexp.QuoteMeta( - `SELECT * FROM "user_identities" WHERE provider=$1 AND provider_uid=$2 ORDER BY "user_identities"."id" LIMIT $3`, + "SELECT * FROM `user_identities` WHERE provider=? AND provider_uid=? ORDER BY `user_identities`.`id` LIMIT ?", )).WithArgs("github", "12345", 1). WillReturnRows(sqlmock.NewRows([]string{"id", "user_id"}).AddRow(int64(7), int64(99))) @@ -31,14 +31,42 @@ func TestCreate(t *testing.T) { ctx, _, mock := hubtest.Database(t) r := NewUserIdentity() + // raw_profile 排在末尾：它带 default，gorm 把有默认值的列放到列表最后。 mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO "user_identities"`)). - WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), - sqlmock.AnyArg(), []byte("{}"), sqlmock.AnyArg(), sqlmock.AnyArg()). + mock.ExpectExec(regexp.QuoteMeta( + "INSERT INTO `user_identities` (`user_id`,`provider`,`provider_uid`,`provider_login`,"+ + "`email`,`createtime`,`updatetime`,`raw_profile`) VALUES (?,?,?,?,?,?,?,?)")). + WithArgs(int64(99), "github", "1", "", "a@b.com", + sqlmock.AnyArg(), sqlmock.AnyArg(), []byte(`{"login":"x"}`)). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + + e := &user_identity_entity.UserIdentity{ + UserID: 99, Provider: "github", ProviderUID: "1", Email: "a@b.com", + RawProfile: []byte(`{"login":"x"}`), + } + assert.NoError(t, r.Create(ctx, e)) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +// raw_profile 的「没有 profile 就存 {}」由 schema 的 DEFAULT ('{}') 表达，仓储层不再 +// 兜一遍：调用方（user_svc.createIdentity）本来就已经规范化过一次，两处兜底意味着 +// 同一个概念有两份实现，而删掉任一处都看不出坏在哪。这里断言仓储层**不写** +// raw_profile 这一列——列缺席时 MySQL 用列默认值填。 +func TestCreate_GivenNoRawProfile_ThenLeavesTheColumnToItsSchemaDefault(t *testing.T) { + ctx, _, mock := hubtest.Database(t) + r := NewUserIdentity() + + mock.ExpectBegin() + mock.ExpectExec(regexp.QuoteMeta( + "INSERT INTO `user_identities` (`user_id`,`provider`,`provider_uid`,`provider_login`,"+ + "`email`,`createtime`,`updatetime`) VALUES (?,?,?,?,?,?,?)")). + WithArgs(int64(99), "github", "1", "", "a@b.com", sqlmock.AnyArg(), sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(1, 1)) mock.ExpectCommit() e := &user_identity_entity.UserIdentity{UserID: 99, Provider: "github", ProviderUID: "1", Email: "a@b.com"} assert.NoError(t, r.Create(ctx, e)) + assert.Empty(t, e.RawProfile, "仓储层不得回填实体字段") assert.NoError(t, mock.ExpectationsWereMet()) } diff --git a/internal/repository/user_repo/user_test.go b/internal/repository/user_repo/user_test.go index 190fa780..b4970ea4 100644 --- a/internal/repository/user_repo/user_test.go +++ b/internal/repository/user_repo/user_test.go @@ -16,7 +16,7 @@ func TestCreate(t *testing.T) { repo := NewUser() mock.ExpectBegin() - mock.ExpectExec(regexp.QuoteMeta(`INSERT INTO "users"`)). + mock.ExpectExec(regexp.QuoteMeta("INSERT INTO `users`")). WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(42, 1)) @@ -33,7 +33,7 @@ func TestFindByEmail_Found(t *testing.T) { ctx, _, mock := hubtest.Database(t) repo := NewUser() - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "users" WHERE email=$1 AND status=$2 ORDER BY "users"."id" LIMIT $3`)). + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `users` WHERE email=? AND status=? ORDER BY `users`.`id` LIMIT ?")). WithArgs("a@b.com", 1, 1). WillReturnRows(sqlmock.NewRows([]string{"id", "email", "status"}).AddRow(int64(1), "a@b.com", 1)) @@ -45,7 +45,7 @@ func TestFindByEmail_Found(t *testing.T) { func TestFindByEmail_NotFound(t *testing.T) { ctx, _, mock := hubtest.Database(t) repo := NewUser() - mock.ExpectQuery(regexp.QuoteMeta(`SELECT * FROM "users" WHERE email=$1 AND status=$2 ORDER BY "users"."id" LIMIT $3`)). + mock.ExpectQuery(regexp.QuoteMeta("SELECT * FROM `users` WHERE email=? AND status=? ORDER BY `users`.`id` LIMIT ?")). WithArgs("missing@x.com", 1, 1). WillReturnRows(sqlmock.NewRows([]string{"id"})) got, err := repo.FindByEmail(ctx, "missing@x.com") diff --git a/internal/testutils/database.go b/internal/testutils/database.go index 2f9ca324..45fa2a14 100644 --- a/internal/testutils/database.go +++ b/internal/testutils/database.go @@ -3,9 +3,6 @@ package testutils import ( "context" - "fmt" - "regexp" - "strings" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -14,26 +11,16 @@ import ( "gorm.io/gorm" ) -type mysqlQueryMatcher struct{} - -func (mysqlQueryMatcher) Match(expectedSQL, actualSQL string) error { - if !strings.Contains(expectedSQL, "`") { - actualSQL = strings.ReplaceAll(actualSQL, "`", `"`) - placeholder := 0 - actualSQL = regexp.MustCompile(`\?`).ReplaceAllStringFunc(actualSQL, func(string) string { - placeholder++ - return fmt.Sprintf("$%d", placeholder) - }) - } - return sqlmock.QueryMatcherRegexp.Match(expectedSQL, actualSQL) -} - // Database 返回带 MySQL 方言 sqlmock 的 ctx + gormDB + mock。 // // db.Ctx(ctx) 自动命中该 mock；其它 redis/cache 组件不影响。 +// +// 期望值按 MySQL 方言原样写：反引号引标识符、`?` 占位符。这里**刻意没有**方言翻译层 +// ——把实际发出的 SQL 改写成别的方言再去匹配，等于让测试断言一段数据库根本不会收到的 +// 文本，读测试的人也会据此以为服务连的是另一种库。 func Database(t *testing.T) (context.Context, *gorm.DB, sqlmock.Sqlmock) { t.Helper() - sqlDB, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(mysqlQueryMatcher{})) + sqlDB, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) if err != nil { t.Fatal(err) } diff --git a/migrations/202605200001_users.go b/migrations/202605200001_users.go index 0efd4139..74bb5eb4 100644 --- a/migrations/202605200001_users.go +++ b/migrations/202605200001_users.go @@ -6,6 +6,18 @@ import ( ) // migration202605200001 创建 users 表。 +// +// email 用 utf8mb4_bin：它是账号的自然键，必须逐字节判等。表默认的 +// utf8mb4_0900_ai_ci 是大小写与重音都不敏感的，唯一键落在它上面意味着 +// "a@b.c" 与 "A@B.C" 算同一个账号——那是一个产品决定，不该由排序规则的默认值 +// 顺手替我们做掉。display_name / avatar_url 是给人看的文本，留默认排序规则。 +// +// active_flag 是 MySQL 表达「部分唯一索引」的写法：唯一键里出现 NULL 的行不参与 +// 约束，所以只有 status=1 的行会互相排斥，等价于 PG 的 +// `CREATE UNIQUE INDEX ... ON users(email) WHERE status = 1`。 +// 键写成 (email, active_flag) 而不是 (active_flag, email)，是为了让同一个索引 +// 既做约束、又能被 user_repo.FindByEmail 的 `WHERE email=?` 当最左前缀用上—— +// 否则 email 上就一个索引都没有，登录路径每次都是全表扫。 func migration202605200001() *gormigrate.Migration { return &gormigrate.Migration{ ID: "202605200001", @@ -13,16 +25,16 @@ func migration202605200001() *gormigrate.Migration { return tx.Exec(` CREATE TABLE users ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, - email varchar(320) NOT NULL, + email varchar(320) COLLATE utf8mb4_bin NOT NULL, email_verified boolean NOT NULL DEFAULT false, display_name varchar(255) NOT NULL DEFAULT '', avatar_url varchar(2048) NOT NULL DEFAULT '', status smallint NOT NULL DEFAULT 1, createtime bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0, - active_email varchar(320) GENERATED ALWAYS AS (IF(status = 1, email, NULL)) STORED, - UNIQUE KEY uk_users_email_active (active_email) - ); + active_flag tinyint GENERATED ALWAYS AS (IF(status = 1, 1, NULL)) STORED, + UNIQUE KEY uk_users_email_active (email, active_flag) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; `).Error }, Rollback: func(tx *gorm.DB) error { diff --git a/migrations/202605200002_user_identities.go b/migrations/202605200002_user_identities.go index c66dd6b5..39873f2d 100644 --- a/migrations/202605200002_user_identities.go +++ b/migrations/202605200002_user_identities.go @@ -5,6 +5,14 @@ import ( "gorm.io/gorm" ) +// migration202605200002 创建 user_identities 表。 +// +// provider / provider_uid / email 是外部身份的自然键，用 utf8mb4_bin 逐字节判等： +// provider_uid 是 OAuth 提供方给的不透明标识，大小写不敏感地判重会把两个不同的 +// 上游账号认成同一个。provider_login 是展示用的用户名，留默认排序规则。 +// +// raw_profile 带上 DEFAULT ('{}')（MySQL 8.0.13+ 的表达式默认值）：让「没有 profile」 +// 这件事由 schema 表达一次，而不是在每个写入方各自兜一遍。 func migration202605200002() *gormigrate.Migration { return &gormigrate.Migration{ ID: "202605200002", @@ -13,16 +21,16 @@ func migration202605200002() *gormigrate.Migration { CREATE TABLE user_identities ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, user_id bigint NOT NULL, - provider varchar(32) NOT NULL, - provider_uid varchar(255) NOT NULL, + provider varchar(32) COLLATE utf8mb4_bin NOT NULL, + provider_uid varchar(255) COLLATE utf8mb4_bin NOT NULL, provider_login varchar(255) NOT NULL DEFAULT '', - email varchar(320) NOT NULL, - raw_profile json NOT NULL, + email varchar(320) COLLATE utf8mb4_bin NOT NULL, + raw_profile json NOT NULL DEFAULT ('{}'), createtime bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0, UNIQUE KEY uk_user_identities_provider_uid (provider, provider_uid), KEY idx_user_identities_user (user_id) - ); + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; `).Error }, Rollback: func(tx *gorm.DB) error { diff --git a/migrations/202605200003_devices.go b/migrations/202605200003_devices.go index daf0c9e6..5403a303 100644 --- a/migrations/202605200003_devices.go +++ b/migrations/202605200003_devices.go @@ -5,6 +5,16 @@ import ( "gorm.io/gorm" ) +// migration202605200003 创建 devices 表。 +// +// fingerprint 是设备的自然键、由桌面端生成，kind 是枚举字面量，两者都用 +// utf8mb4_bin：指纹大小写不敏感地判重会把两台不同的机器认成同一台，进而让第二台 +// 的注册撞上 uk_devices_user_fingerprint。name 是用户可改的展示名，留默认排序规则。 +// +// idx_devices_user_active 是普通复合索引而不是部分索引：PG 那边写的是 +// `WHERE status = 1`，MySQL 没有部分索引，但把 status 放进键里同样能服务 +// `WHERE user_id=? AND status=?`，只是索引会连非活跃行一起收——设备表很小，不值得 +// 为此再加一个生成列。 func migration202605200003() *gormigrate.Migration { return &gormigrate.Migration{ ID: "202605200003", @@ -14,17 +24,17 @@ func migration202605200003() *gormigrate.Migration { id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, user_id bigint NOT NULL, name varchar(255) NOT NULL, - kind varchar(32) NOT NULL, + kind varchar(32) COLLATE utf8mb4_bin NOT NULL, platform varchar(64) NOT NULL DEFAULT '', version varchar(64) NOT NULL DEFAULT '', - fingerprint varchar(255) NOT NULL, + fingerprint varchar(255) COLLATE utf8mb4_bin NOT NULL, last_seen_at bigint NOT NULL DEFAULT 0, status smallint NOT NULL DEFAULT 1, createtime bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0, UNIQUE KEY uk_devices_user_fingerprint (user_id, fingerprint), KEY idx_devices_user_active (user_id, status) - ); + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; `).Error }, Rollback: func(tx *gorm.DB) error { diff --git a/migrations/202605200004_device_tokens.go b/migrations/202605200004_device_tokens.go index e6ebfbf4..164b936e 100644 --- a/migrations/202605200004_device_tokens.go +++ b/migrations/202605200004_device_tokens.go @@ -5,6 +5,15 @@ import ( "gorm.io/gorm" ) +// migration202605200004 创建 device_tokens 表。 +// +// refresh_token_hash 是 sha256 的十六进制（device_svc 里 hex.EncodeToString，恒为 +// 64 位小写），access_jti 是 ULID，ip 是点分/冒号文本：三者都是机器生成的标识， +// 用 utf8mb4_bin 逐字节判等。尤其 refresh_token_hash 上挂着唯一键， +// 大小写不敏感会让两个不同的哈希互相顶掉。user_agent 是外部原文，留默认排序规则。 +// +// idx_dtokens_device_active 把 revoked_at 放进键里代替 PG 的 `WHERE revoked_at = 0`， +// 理由同 devices：一条复合索引就能服务查询，不必为此加生成列。 func migration202605200004() *gormigrate.Migration { return &gormigrate.Migration{ ID: "202605200004", @@ -13,18 +22,18 @@ func migration202605200004() *gormigrate.Migration { CREATE TABLE device_tokens ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, device_id bigint NOT NULL, - access_jti varchar(255) NOT NULL DEFAULT '', - refresh_token_hash varchar(64) NOT NULL, + access_jti varchar(64) COLLATE utf8mb4_bin NOT NULL DEFAULT '', + refresh_token_hash varchar(64) COLLATE utf8mb4_bin NOT NULL, refresh_expires_at bigint NOT NULL DEFAULT 0, last_used_at bigint NOT NULL DEFAULT 0, rotated_from_id bigint NOT NULL DEFAULT 0, revoked_at bigint NOT NULL DEFAULT 0, user_agent varchar(512) NOT NULL DEFAULT '', - ip varchar(45), + ip varchar(45) COLLATE utf8mb4_bin, createtime bigint NOT NULL DEFAULT 0, UNIQUE KEY uk_dtokens_refresh_hash (refresh_token_hash), KEY idx_dtokens_device_active (device_id, revoked_at) - ); + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; `).Error }, Rollback: func(tx *gorm.DB) error { diff --git a/migrations/202605200005_device_flow_codes.go b/migrations/202605200005_device_flow_codes.go index 0a7981e9..98d4bfbc 100644 --- a/migrations/202605200005_device_flow_codes.go +++ b/migrations/202605200005_device_flow_codes.go @@ -5,16 +5,33 @@ import ( "gorm.io/gorm" ) +// migration202605200005 创建 device_flow_codes 表（RFC 8628 的 device_code / +// user_code 状态机）。 +// +// device_code 与 user_code 都是凭据，用 utf8mb4_bin 逐字节判等：默认的 +// utf8mb4_0900_ai_ci 会让 device_code 的比较大小写不敏感，等于凭空放宽一个 bearer +// 凭据的匹配条件。两者的长度按生成器实际产出来定，不留无意义的余量—— +// device_code 是 randomBase32(32)（32 字节 base32、无填充，恒为 52 位小写）， +// user_code 是 usercode.Generate() 的 "XXX-XXX"（7 位大写）。device_code 还是主键， +// InnoDB 会把主键塞进每一条二级索引，所以它的宽度是真实成本，不能随手写 varchar(255)。 +// +// pending_flag 是 MySQL 表达「部分唯一索引」的写法，等价于 PG 的 +// `... ON device_flow_codes(user_code) WHERE consumed_at = 0 AND denied_at = 0`： +// 唯一键里出现 NULL 的行不参与约束，因此只有未结算的行会互相排斥。 +// 键写成 (user_code, pending_flag) 是为了让同一个索引也能被 +// `WHERE user_code=? AND consumed_at=0 AND denied_at=0` 当最左前缀用上——这条路径 +// 是设备每 5 秒一次的轮询和 approve/deny 两条 UPDATE，没有索引就是全表扫， +// 而全表扫的 UPDATE 在 InnoDB 下还会把 next-key 锁铺满整张表。 func migration202605200005() *gormigrate.Migration { return &gormigrate.Migration{ ID: "202605200005", Migrate: func(tx *gorm.DB) error { return tx.Exec(` CREATE TABLE device_flow_codes ( - device_code varchar(255) PRIMARY KEY, - user_code varchar(32) NOT NULL, - device_kind varchar(32) NOT NULL, - client_fingerprint varchar(255) NOT NULL, + device_code varchar(64) COLLATE utf8mb4_bin PRIMARY KEY, + user_code varchar(16) COLLATE utf8mb4_bin NOT NULL, + device_kind varchar(32) COLLATE utf8mb4_bin NOT NULL, + client_fingerprint varchar(255) COLLATE utf8mb4_bin NOT NULL, platform varchar(64) NOT NULL DEFAULT '', version varchar(64) NOT NULL DEFAULT '', authorized_user_id bigint NOT NULL DEFAULT 0, @@ -25,11 +42,11 @@ func migration202605200005() *gormigrate.Migration { last_polled_at bigint NOT NULL DEFAULT 0, expires_at bigint NOT NULL DEFAULT 0, createtime bigint NOT NULL DEFAULT 0, - pending_user_code varchar(32) GENERATED ALWAYS AS - (IF(consumed_at = 0 AND denied_at = 0, user_code, NULL)) STORED, - UNIQUE KEY uk_dfc_user_code_pending (pending_user_code), + pending_flag tinyint GENERATED ALWAYS AS + (IF(consumed_at = 0 AND denied_at = 0, 1, NULL)) STORED, + UNIQUE KEY uk_dfc_user_code_pending (user_code, pending_flag), KEY idx_dfc_expires (expires_at) - ); + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; `).Error }, Rollback: func(tx *gorm.DB) error { diff --git a/migrations/202608090001_workspace_sync.go b/migrations/202608090001_workspace_sync.go index 196eb417..3ad3857a 100644 --- a/migrations/202608090001_workspace_sync.go +++ b/migrations/202608090001_workspace_sync.go @@ -11,6 +11,31 @@ import ( // 同步组（sync_objects）与上报组（device_local_paths）语义不同，所以不是一张表： // 同步组双向同步、有版本号与墓碑；上报组按设备分命名空间、整份快照替换， // 没有删除时间也没有冲突元数据。 +// +// sync_id / project_sync_id / agentred_fingerprint / kind 都是客户端自带的不透明 +// 标识，一律 utf8mb4_bin 逐字节判等。表默认的 utf8mb4_0900_ai_ci 大小写不敏感， +// 会让 "abc" 与 "ABC" 两个不同的 sync_id 撞上 uk_sync_objects_identity， +// 而且 `WHERE sync_id=?` 会取回另一行——同步的一切都建立在这个标识精确可比上。 +// +// uk_sync_objects_location 是 agentred 路径的账号内自然键，只约束存活的行：墓碑不占 +// 自然键，否则删掉再建就建不回来。用 live_location_flag（存活时为 1、否则为 NULL） +// 把墓碑摘出去——唯一键里出现 NULL 的行不参与约束，这正是 MySQL 表达部分唯一索引的 +// 写法，等价于 PG 那条带 `WHERE kind = 'project_location' AND deleted_at = 0` 的 +// (user_id, project_sync_id, agentred_fingerprint) 部分唯一索引。 +// +// 键里放的是三个真列而不是把它们拼成一个字符串：拼接需要一个分隔符，而在 +// utf8mb4_0900_ai_ci 下 CHAR(0) 这类控制字符的排序权重为空、会被直接忽略， +// ('proj','Xdev') 与 ('projX','dev') 会拼成同一个键、误判成重复。放真列既没有这个 +// 问题，又能让 objectRepo.FindLocationByNaturalKey 走 +// (user_id, project_sync_id, agentred_fingerprint) 这个最左前缀。 +// +// idx_sync_objects_fingerprint 让指纹能直接 join 到 devices.fingerprint：web 控制台 +// 不需要额外的映射表就能说出「这条配置属于哪台机器」。PG 那边它带一个「指纹非空」的 +// 条件，MySQL 这里收全部行——多收的是空指纹那一批，不值得为此再加一个生成列。 +// +// sync_avatars.content 用 mediumtext 而不是 text：MySQL 的 text 上限是 65535 字节， +// 而 sync_svc.MaxAvatarBytes 允许 4 MiB，用 text 会让超过 64 KB 的头像直接写不进去 +// （ER_DATA_TOO_LONG）。mediumtext 是 16 MiB，覆盖得住那个上限。 func migration202608090001() *gormigrate.Migration { return &gormigrate.Migration{ ID: "202608090001", @@ -19,54 +44,55 @@ func migration202608090001() *gormigrate.Migration { CREATE TABLE sync_objects ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, user_id bigint NOT NULL, - kind varchar(32) NOT NULL, - sync_id varchar(255) NOT NULL, - project_sync_id varchar(255) NOT NULL DEFAULT '', - agentred_fingerprint varchar(255) NOT NULL DEFAULT '', - payload json NOT NULL, + kind varchar(32) COLLATE utf8mb4_bin NOT NULL, + sync_id varchar(255) COLLATE utf8mb4_bin NOT NULL, + project_sync_id varchar(255) COLLATE utf8mb4_bin NOT NULL DEFAULT '', + agentred_fingerprint varchar(255) COLLATE utf8mb4_bin NOT NULL DEFAULT '', + payload json NOT NULL DEFAULT ('{}'), version bigint NOT NULL, sync_updated_at bigint NOT NULL DEFAULT 0, source_device_id bigint NOT NULL DEFAULT 0, deleted_at bigint NOT NULL DEFAULT 0, createtime bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0, - live_location_key varchar(511) GENERATED ALWAYS AS - (IF(kind = 'project_location' AND deleted_at = 0, - CONCAT(project_sync_id, CHAR(0), agentred_fingerprint), NULL)) STORED, + -- 存活标志，只用来把墓碑从下面那个唯一键里摘出去（见函数注释）。 + live_location_flag tinyint GENERATED ALWAYS AS + (IF(kind = 'project_location' AND deleted_at = 0, 1, NULL)) STORED, UNIQUE KEY uk_sync_objects_identity (user_id, sync_id), - UNIQUE KEY uk_sync_objects_location (user_id, live_location_key), + UNIQUE KEY uk_sync_objects_location + (user_id, project_sync_id, agentred_fingerprint, live_location_flag), KEY idx_sync_objects_cursor (user_id, version), KEY idx_sync_objects_fingerprint (user_id, agentred_fingerprint) - )`, ` + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, ` CREATE TABLE sync_account_seqs ( user_id bigint PRIMARY KEY, version_seq bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0 - )`, ` + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, ` CREATE TABLE sync_device_states ( user_id bigint NOT NULL, device_id bigint NOT NULL, last_sync_at bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0, PRIMARY KEY (user_id, device_id) - )`, ` + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, ` CREATE TABLE sync_avatars ( user_id bigint NOT NULL, - content_hash varchar(64) NOT NULL, + content_hash varchar(64) COLLATE utf8mb4_bin NOT NULL, content_type varchar(255) NOT NULL DEFAULT '', - content text NOT NULL, + content mediumtext NOT NULL, byte_size bigint NOT NULL DEFAULT 0, createtime bigint NOT NULL DEFAULT 0, PRIMARY KEY (user_id, content_hash) - )`, ` + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, ` CREATE TABLE device_local_paths ( user_id bigint NOT NULL, device_id bigint NOT NULL, - project_sync_id varchar(255) NOT NULL, + project_sync_id varchar(255) COLLATE utf8mb4_bin NOT NULL, path text NOT NULL, updatetime bigint NOT NULL DEFAULT 0, PRIMARY KEY (user_id, device_id, project_sync_id) - )`, + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, } for _, statement := range statements { if err := tx.Exec(statement).Error; err != nil { diff --git a/migrations/202608100001_followed_sessions.go b/migrations/202608100001_followed_sessions.go index 39eada81..aba92a45 100644 --- a/migrations/202608100001_followed_sessions.go +++ b/migrations/202608100001_followed_sessions.go @@ -10,8 +10,12 @@ import ( // 这是 agentre-server 本轮唯一的服务端新增表，也是硬不变量（server 不持有任何 // 会话内容）的唯一例外，且它存的是「指向」——目标设备指纹 + 会话标识 + 关注时间， // 不含标题、消息或转录。表按 (user_id, device_fingerprint, session_id) 唯一， -// 关注/取消因此幂等：重复关注命中唯一索引时 ON DUPLICATE KEY DO NOTHING，不新增行、 +// 关注/取消因此幂等：重复关注命中唯一索引时那条 INSERT 什么都不改（gorm 的 DoNothing +// 在 MySQL 下发出 ON DUPLICATE KEY UPDATE 的自赋值形式），不新增行、 // 不重置首次关注时间；取消就是一条 DELETE，删不到也是成功。 +// +// device_fingerprint 与 session_id 是目标设备与会话的不透明标识，用 utf8mb4_bin +// 逐字节判等：大小写不敏感会把两个不同的会话认成同一个，名单就指错了对象。 func migration202608100001() *gormigrate.Migration { return &gormigrate.Migration{ ID: "202608100001", @@ -20,14 +24,14 @@ func migration202608100001() *gormigrate.Migration { CREATE TABLE followed_sessions ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, user_id bigint NOT NULL, - device_fingerprint varchar(255) NOT NULL, - session_id varchar(255) NOT NULL, + device_fingerprint varchar(255) COLLATE utf8mb4_bin NOT NULL, + session_id varchar(255) COLLATE utf8mb4_bin NOT NULL, followed_at bigint NOT NULL DEFAULT 0, createtime bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0, UNIQUE KEY uk_followed_sessions_identity (user_id, device_fingerprint, session_id) - ); + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; `).Error }, Rollback: func(tx *gorm.DB) error { diff --git a/migrations/migrations.go b/migrations/migrations.go index 84a1203e..06a1c612 100644 --- a/migrations/migrations.go +++ b/migrations/migrations.go @@ -43,12 +43,12 @@ func RunMigrations(db *gorm.DB) error { }) } -// withMigrationLock 在持有迁移 advisory lock 期间执行 fn。 +// withMigrationLock 在持有迁移 named lock 期间执行 fn。 // // named lock 是会话级的，而 gorm 的 *gorm.DB 从连接池里取连接：如果直接用 // db.Exec("GET_LOCK") 加锁，锁可能落在连接 A 上，随后 fn 里的迁移语句却从 // 连接 B 跑，锁在归还连接池的那一刻就形同虚设。因此这里用 sqlDB.Conn(ctx) 单独要一条 -// 连接、全程只用它做加锁/解锁；fn 内部的迁移仍然照常走连接池，advisory lock 不绑定数据， +// 连接、全程只用它做加锁/解锁；fn 内部的迁移仍然照常走连接池，named lock 不绑定数据， // 这样是安全的。副本崩溃时这条专用连接断开，MySQL 会自动释放锁，无需人工介入。 func withMigrationLock(db *gorm.DB, fn func() error) error { sqlDB, err := db.DB() @@ -73,16 +73,22 @@ func withMigrationLock(db *gorm.DB, fn func() error) error { // acquireMigrationLock 轮询 GET_LOCK 直到拿到锁或等待预算耗尽。 // -// 用零超时 GET_LOCK 轮询，是为了让等待有明确上界、可观测：拿不到锁时立刻 -// 返回，不会让副本静默挂起到被存活探针杀掉。 +// 超时参数写 0 而不是让 GET_LOCK 自己阻塞，是为了让等待有明确上界、可观测：拿不到锁 +// 时立刻返回，不会让副本静默挂起到被存活探针杀掉。 +// +// 返回值扫进 sql.NullInt64 而不是 int：GET_LOCK 有三种结果——1 拿到、0 没拿到、 +// NULL 发生了错误（连接被杀、锁名超长）。只有 1 算拿到，另外两种都继续轮询、最终以 +// 那条「等迁移锁超时」的错误收场。直接扫进 int 的话，NULL 会变成一个 +// 「converting NULL to int is unsupported」的驱动错误——那条消息既不说明是等锁失败， +// 也会把排查引向「扫描类型」这个完全无关的方向。 func acquireMigrationLock(ctx context.Context, conn *sql.Conn) error { deadline := time.Now().Add(migrationLockWaitBudget) for { - var locked int + var locked sql.NullInt64 if err := conn.QueryRowContext(ctx, "SELECT GET_LOCK(?, 0)", migrationLockName).Scan(&locked); err != nil { return fmt.Errorf("migrations: try named lock: %w", err) } - if locked == 1 { + if locked.Valid && locked.Int64 == 1 { return nil } if !time.Now().Before(deadline) { @@ -95,8 +101,11 @@ func acquireMigrationLock(ctx context.Context, conn *sql.Conn) error { // releaseMigrationLock 解锁；专用连接紧接着就会被关闭，即便这里失败，MySQL 也会在 // 连接断开时释放这条会话级的锁，因此这里只记日志、不把错误抛给调用方。 +// +// 同样扫进 sql.NullInt64：RELEASE_LOCK 在锁不存在时返回 NULL，扫进 int 会得到一条 +// 与「解锁」毫无关系的类型转换告警。 func releaseMigrationLock(ctx context.Context, conn *sql.Conn) { - var unlocked int + var unlocked sql.NullInt64 if err := conn.QueryRowContext(ctx, "SELECT RELEASE_LOCK(?)", migrationLockName).Scan(&unlocked); err != nil { logger.Ctx(ctx).Warn("release migration lock", zap.Error(err)) } diff --git a/migrations/migrations_test.go b/migrations/migrations_test.go new file mode 100644 index 00000000..f9894b89 --- /dev/null +++ b/migrations/migrations_test.go @@ -0,0 +1,126 @@ +package migrations + +import ( + "regexp" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" + + hubtest "agentre-server/internal/testutils" +) + +// withPatchedLockTiming 把轮询间隔和等待预算换成测试专用的小值，跑完自动还原， +// 避免真跑迁移锁默认的 120s 预算拖慢测试。 +func withPatchedLockTiming(t *testing.T, interval, budget time.Duration) { + t.Helper() + origInterval, origBudget := migrationLockPollInterval, migrationLockWaitBudget + migrationLockPollInterval, migrationLockWaitBudget = interval, budget + t.Cleanup(func() { + migrationLockPollInterval, migrationLockWaitBudget = origInterval, origBudget + }) +} + +// TestWithMigrationLock_RetriesUntilAcquired 断言：拿不到锁时会重试（GET_LOCK 的零 +// 超时形式返回 0），拿到锁（返回 1）后才跑传入的迁移函数，结束后 RELEASE_LOCK。 +func TestWithMigrationLock_RetriesUntilAcquired(t *testing.T) { + withPatchedLockTiming(t, time.Millisecond, 120*time.Second) + _, gormDB, mock := hubtest.Database(t) + + mock.ExpectQuery(regexp.QuoteMeta("GET_LOCK")). + WillReturnRows(sqlmock.NewRows([]string{"GET_LOCK(?, 0)"}).AddRow(0)) + mock.ExpectQuery(regexp.QuoteMeta("GET_LOCK")). + WillReturnRows(sqlmock.NewRows([]string{"GET_LOCK(?, 0)"}).AddRow(1)) + mock.ExpectQuery(regexp.QuoteMeta("RELEASE_LOCK")). + WillReturnRows(sqlmock.NewRows([]string{"RELEASE_LOCK(?)"}).AddRow(1)) + + ran := false + err := withMigrationLock(gormDB, func() error { + ran = true + return nil + }) + + assert.NoError(t, err) + assert.True(t, ran, "migration func should run once the lock is acquired") + assert.NoError(t, mock.ExpectationsWereMet()) +} + +// TestWithMigrationLock_PropagatesMigrateError 断言：迁移函数本身的失败会被如实返回， +// 但锁依然会在结束时释放（不会把连接晾在持锁状态）。 +func TestWithMigrationLock_PropagatesMigrateError(t *testing.T) { + withPatchedLockTiming(t, time.Millisecond, 120*time.Second) + _, gormDB, mock := hubtest.Database(t) + + mock.ExpectQuery(regexp.QuoteMeta("GET_LOCK")). + WillReturnRows(sqlmock.NewRows([]string{"GET_LOCK(?, 0)"}).AddRow(1)) + mock.ExpectQuery(regexp.QuoteMeta("RELEASE_LOCK")). + WillReturnRows(sqlmock.NewRows([]string{"RELEASE_LOCK(?)"}).AddRow(1)) + + wantErr := assert.AnError + err := withMigrationLock(gormDB, func() error { + return wantErr + }) + + assert.ErrorIs(t, err, wantErr) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +// TestWithMigrationLock_TimesOut 断言：轮询超过等待预算后返回一个说明「等迁移锁超时」的 +// 错误，且从未拿到锁，因此从不会调用传入的迁移函数，也不会尝试解锁。 +func TestWithMigrationLock_TimesOut(t *testing.T) { + const ( + interval = 5 * time.Millisecond + budget = 20 * time.Millisecond + ) + withPatchedLockTiming(t, interval, budget) + _, gormDB, mock := hubtest.Database(t) + + // 预算/间隔比很小，实际重试次数以真实时钟为准，注册一批足够多的「未拿到锁」响应。 + for i := 0; i < 50; i++ { + mock.ExpectQuery(regexp.QuoteMeta("GET_LOCK")). + WillReturnRows(sqlmock.NewRows([]string{"GET_LOCK(?, 0)"}).AddRow(0)) + } + + start := time.Now() + ran := false + err := withMigrationLock(gormDB, func() error { + ran = true + return nil + }) + elapsed := time.Since(start) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "timed out") + assert.Contains(t, err.Error(), "migration lock") + assert.False(t, ran, "migration func must not run without the lock") + assert.GreaterOrEqual(t, elapsed, budget, "must have actually waited out the budget via retries") +} + +// TestWithMigrationLock_TreatsNullAsNotAcquired 断言：GET_LOCK 返回 NULL 时按「没拿到 +// 锁」处理——继续轮询，最终以那条「等迁移锁超时」的错误收场。 +// +// 这是 pg_try_advisory_lock 没有的形态：它只返回 true/false，而 GET_LOCK 在发生错误时 +// （连接被杀、锁名超长）返回 NULL。把 NULL 直接扫进 int 会得到一个 +// 「converting NULL to int is unsupported」的驱动错误——那条消息既不说明是等锁失败， +// 也让运维顺着「扫描类型」这个完全无关的方向去查。 +func TestWithMigrationLock_TreatsNullAsNotAcquired(t *testing.T) { + withPatchedLockTiming(t, time.Millisecond, 10*time.Millisecond) + _, gormDB, mock := hubtest.Database(t) + + for i := 0; i < 50; i++ { + mock.ExpectQuery(regexp.QuoteMeta("GET_LOCK")). + WillReturnRows(sqlmock.NewRows([]string{"GET_LOCK(?, 0)"}).AddRow(nil)) + } + + ran := false + err := withMigrationLock(gormDB, func() error { + ran = true + return nil + }) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "timed out", "NULL 要走等锁超时那条路，而不是驱动的扫描错误") + assert.Contains(t, err.Error(), "migration lock") + assert.False(t, ran, "NULL 不是「拿到锁」，迁移绝不能在这种情况下开跑") +} From 6437d8ea8b0f939ee1d4415d9c29ea904214f0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E4=B8=80=E4=B9=8B?= Date: Thu, 13 Aug 2026 11:26:09 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=90=9B=20database:=20email/user=5Fcod?= =?UTF-8?q?e=20=E6=94=B9=E4=B8=BA=E5=A4=A7=E5=B0=8F=E5=86=99=E4=B8=8D?= =?UTF-8?q?=E6=95=8F=E6=84=9F=EF=BC=8C=E5=B9=B6=E6=8A=8A=E6=A0=87=E8=AF=86?= =?UTF-8?q?=E5=88=97=E6=8D=A2=E5=88=B0=20NO=20PAD=20=E6=8E=92=E5=BA=8F?= =?UTF-8?q?=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 上一版对「什么算相等」做了两个错误决定，都由排序规则的选择静默生效。 1) email 不该区分大小写。同一个人用 A@b.C 和 a@b.c 注册会落成两个账号。改成 utf8mb4_0900_as_ci：只折叠大小写。没有直接用表默认的 ai_ci，因为 ai = accent-insensitive 连重音一起折叠，会把 e@x.c 与 é@x.c 当成同一个邮箱。 users.email 与 user_identities.email 必须同规则，否则两列比较会报 illegal mix。 2) user_code 同理，它是印给人看、由人敲进浏览器的。usercode.Normalize 已经会转大写， 排序规则是第二层保障：将来多一条忘了 Normalize 的路径，症状会是「查不到这个验证码」 这种很难联想到大小写的报错。 3) 顺带查出来的同类问题：utf8mb4_bin 是 PAD SPACE，会忽略尾随空格——'x' 与 'x ' 在它下面相等，于是尾随空格不同的两个 sync_id 会撞唯一键、WHERE sync_id='x' 还会 取回另一行。这跟上一轮修掉的 CHAR(0) 是同一类缺陷，而 PG 的 text 是 NO PAD。 全部标识列换成 utf8mb4_0900_bin（既逐字节、又 NO PAD）。 4) ip / user_agent 只写不读，从不出现在任何 WHERE 里，去掉它们的显式排序规则。只在 真正参与比较的列上写，读的人才知道哪些列的判等语义是被刻意选过的。 新增 migrations/collation_test.go 把策略钉住：DDL 里出现任何 PAD SPACE 排序规则即失败， 人手输入的标识符必须 _ci、不透明标识必须 _0900_bin、每张表必须钉 ENGINE 与字符集。 这类缺陷在单测和肉眼 review 里都看不出来，只能靠断言挡。 排序规则策略连同两个坑（_0900_ 之外都是 PAD SPACE；ai_ci 不只是忽略大小写）写进 docs/architecture.md，并列出必须共享排序规则的两组跨表比较列。 实库验证：email 大小写互撞而重音不撞、小写验证码能对上、'x' 与 'x ' 互不冲突、 三组跨表 join 不报 illegal mix；上一轮四个修复与三条索引路径全部回归通过。 --- docs/architecture.md | 26 +++++ migrations/202605200001_users.go | 21 +++- migrations/202605200002_user_identities.go | 17 +-- migrations/202605200003_devices.go | 9 +- migrations/202605200004_device_tokens.go | 16 +-- migrations/202605200005_device_flow_codes.go | 20 ++-- migrations/202608090001_workspace_sync.go | 22 ++-- migrations/202608100001_followed_sessions.go | 9 +- migrations/collation_test.go | 107 +++++++++++++++++++ 9 files changed, 207 insertions(+), 40 deletions(-) create mode 100644 migrations/collation_test.go diff --git a/docs/architecture.md b/docs/architecture.md index c65a2ec2..91f2565b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -103,6 +103,32 @@ err := db.Ctx(ctx).Transaction(func(tx *gorm.DB) error { A repository that reaches for `db.Default()` silently escapes the transaction. +### Column collations are part of the contract + +A collation decides what "equal" means, so on any column that a `WHERE`, a `JOIN` or a +unique key compares, it is a behavioural choice, not formatting. Pick it explicitly: + +| Kind of value | Collation | Why | +| --- | --- | --- | +| Opaque identifiers, hashes, bearer credentials — `sync_id`, `*_fingerprint`, `session_id`, `device_code`, `refresh_token_hash`, `content_hash`, `provider_uid`, enum-ish `kind` | `utf8mb4_0900_bin` | Two values differing in any byte are two different things. Folding them merges distinct records and widens credential matching. | +| Identifiers a human types — `email`, `user_code` | `utf8mb4_0900_as_ci` | Case must not matter: one mailbox is one account, and a code typed lowercase must still match. | +| Text that is only stored and displayed — `display_name`, `name`, `platform`, `version`, `user_agent`, `ip`, `path`, `content_type` | *(table default `utf8mb4_0900_ai_ci`)* | Never compared, so the choice is inert. Leaving it unset marks it as "not load-bearing". | + +Two traps, both of which produced real bugs in the PostgreSQL→MySQL move: + +- **Only use the `utf8mb4_0900_*` family.** `utf8mb4_bin` and `utf8mb4_general_ci` are + `PAD SPACE`, so they ignore trailing spaces — `'x'` equals `'x '`, and two `sync_id`s + differing only by a trailing space collide on the unique key. Every `_0900_` collation is + `NO PAD`, which is what PostgreSQL `text` does. + `migrations/collation_test.go` fails the build if a `PAD SPACE` collation appears in DDL. +- **`ai_ci` is not "case-insensitive", it is also accent-insensitive.** As an email + collation it makes `e@x.c` and `é@x.c` the same address. `as_ci` is the case-only tier. + +Columns compared against each other must share a collation, or MySQL raises *illegal mix of +collations* at query time: `devices.fingerprint`, `sync_objects.agentred_fingerprint`, +`followed_sessions.device_fingerprint` and `device_flow_codes.client_fingerprint` are one +such group; `users.email` and `user_identities.email` are another. + ## Routing and auth shapes `internal/api/router.go` is the one place the whole route tree is visible, and the diff --git a/migrations/202605200001_users.go b/migrations/202605200001_users.go index 74bb5eb4..7d90cec5 100644 --- a/migrations/202605200001_users.go +++ b/migrations/202605200001_users.go @@ -7,10 +7,21 @@ import ( // migration202605200001 创建 users 表。 // -// email 用 utf8mb4_bin：它是账号的自然键，必须逐字节判等。表默认的 -// utf8mb4_0900_ai_ci 是大小写与重音都不敏感的，唯一键落在它上面意味着 -// "a@b.c" 与 "A@B.C" 算同一个账号——那是一个产品决定，不该由排序规则的默认值 -// 顺手替我们做掉。display_name / avatar_url 是给人看的文本，留默认排序规则。 +// email 用 utf8mb4_0900_as_ci：**大小写不敏感、但不折叠重音**。 +// +// 大小写不敏感是产品决定：同一个人用 "A@b.C" 和 "a@b.c" 注册必须落在同一个账号上， +// 否则同一个邮箱能注册出两个账号。既然唯一键与 FindByEmail 都走这一列，把这件事交给 +// 排序规则比在每个写入方各自 lower() 一遍更可靠——少一处就漏一处。 +// +// 但不能图省事直接用表默认的 utf8mb4_0900_ai_ci：ai = accent-insensitive，它连重音 +// 都折叠，会把 e@x.c 与 é@x.c 当成同一个邮箱，而那是两个不同的收件人。as_ci 正好是 +// 「只折叠大小写」这一档。 +// +// display_name / avatar_url 从不参与比较，留表默认排序规则即可。 +// +// 注意所有显式排序规则都取 _0900_ 那一族，因为它们是 NO PAD。老的 utf8mb4_bin / +// utf8mb4_general_ci 是 PAD SPACE，会忽略尾随空格（'a@b.c ' 等于 'a@b.c'）， +// 那和 PG 的 text 语义不一样。 // // active_flag 是 MySQL 表达「部分唯一索引」的写法：唯一键里出现 NULL 的行不参与 // 约束，所以只有 status=1 的行会互相排斥，等价于 PG 的 @@ -25,7 +36,7 @@ func migration202605200001() *gormigrate.Migration { return tx.Exec(` CREATE TABLE users ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, - email varchar(320) COLLATE utf8mb4_bin NOT NULL, + email varchar(320) COLLATE utf8mb4_0900_as_ci NOT NULL, email_verified boolean NOT NULL DEFAULT false, display_name varchar(255) NOT NULL DEFAULT '', avatar_url varchar(2048) NOT NULL DEFAULT '', diff --git a/migrations/202605200002_user_identities.go b/migrations/202605200002_user_identities.go index 39873f2d..3eb006b5 100644 --- a/migrations/202605200002_user_identities.go +++ b/migrations/202605200002_user_identities.go @@ -7,9 +7,14 @@ import ( // migration202605200002 创建 user_identities 表。 // -// provider / provider_uid / email 是外部身份的自然键，用 utf8mb4_bin 逐字节判等： -// provider_uid 是 OAuth 提供方给的不透明标识，大小写不敏感地判重会把两个不同的 -// 上游账号认成同一个。provider_login 是展示用的用户名，留默认排序规则。 +// provider / provider_uid 用 utf8mb4_0900_bin 逐字节判等：provider_uid 是 OAuth 提供方 +// 给的不透明标识，大小写不敏感地判重会把两个不同的上游账号认成同一个。 +// +// email 跟 users.email 保持同一个排序规则（utf8mb4_0900_as_ci，大小写不敏感、不折叠 +// 重音）：两列语义相同，排序规则也必须相同——不同排序规则的两列直接比较会被 MySQL +// 判为 illegal mix of collations 而报错。 +// +// provider_login 是展示用的用户名，留表默认排序规则。 // // raw_profile 带上 DEFAULT ('{}')（MySQL 8.0.13+ 的表达式默认值）：让「没有 profile」 // 这件事由 schema 表达一次，而不是在每个写入方各自兜一遍。 @@ -21,10 +26,10 @@ func migration202605200002() *gormigrate.Migration { CREATE TABLE user_identities ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, user_id bigint NOT NULL, - provider varchar(32) COLLATE utf8mb4_bin NOT NULL, - provider_uid varchar(255) COLLATE utf8mb4_bin NOT NULL, + provider varchar(32) COLLATE utf8mb4_0900_bin NOT NULL, + provider_uid varchar(255) COLLATE utf8mb4_0900_bin NOT NULL, provider_login varchar(255) NOT NULL DEFAULT '', - email varchar(320) COLLATE utf8mb4_bin NOT NULL, + email varchar(320) COLLATE utf8mb4_0900_as_ci NOT NULL, raw_profile json NOT NULL DEFAULT ('{}'), createtime bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0, diff --git a/migrations/202605200003_devices.go b/migrations/202605200003_devices.go index 5403a303..cac03c84 100644 --- a/migrations/202605200003_devices.go +++ b/migrations/202605200003_devices.go @@ -8,9 +8,12 @@ import ( // migration202605200003 创建 devices 表。 // // fingerprint 是设备的自然键、由桌面端生成，kind 是枚举字面量，两者都用 -// utf8mb4_bin：指纹大小写不敏感地判重会把两台不同的机器认成同一台，进而让第二台 +// utf8mb4_0900_bin：指纹大小写不敏感地判重会把两台不同的机器认成同一台，进而让第二台 // 的注册撞上 uk_devices_user_fingerprint。name 是用户可改的展示名，留默认排序规则。 // +// fingerprint 会被 sync_objects.agentred_fingerprint 与 followed_sessions +// .device_fingerprint 拿去比较，那两列必须用同一个排序规则。 +// // idx_devices_user_active 是普通复合索引而不是部分索引：PG 那边写的是 // `WHERE status = 1`，MySQL 没有部分索引，但把 status 放进键里同样能服务 // `WHERE user_id=? AND status=?`，只是索引会连非活跃行一起收——设备表很小，不值得 @@ -24,10 +27,10 @@ func migration202605200003() *gormigrate.Migration { id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, user_id bigint NOT NULL, name varchar(255) NOT NULL, - kind varchar(32) COLLATE utf8mb4_bin NOT NULL, + kind varchar(32) COLLATE utf8mb4_0900_bin NOT NULL, platform varchar(64) NOT NULL DEFAULT '', version varchar(64) NOT NULL DEFAULT '', - fingerprint varchar(255) COLLATE utf8mb4_bin NOT NULL, + fingerprint varchar(255) COLLATE utf8mb4_0900_bin NOT NULL, last_seen_at bigint NOT NULL DEFAULT 0, status smallint NOT NULL DEFAULT 1, createtime bigint NOT NULL DEFAULT 0, diff --git a/migrations/202605200004_device_tokens.go b/migrations/202605200004_device_tokens.go index 164b936e..6ea6d3a3 100644 --- a/migrations/202605200004_device_tokens.go +++ b/migrations/202605200004_device_tokens.go @@ -8,9 +8,13 @@ import ( // migration202605200004 创建 device_tokens 表。 // // refresh_token_hash 是 sha256 的十六进制（device_svc 里 hex.EncodeToString，恒为 -// 64 位小写），access_jti 是 ULID，ip 是点分/冒号文本：三者都是机器生成的标识， -// 用 utf8mb4_bin 逐字节判等。尤其 refresh_token_hash 上挂着唯一键， -// 大小写不敏感会让两个不同的哈希互相顶掉。user_agent 是外部原文，留默认排序规则。 +// 64 位小写），access_jti 是 ULID：两者都是机器生成的凭据/标识，用 +// utf8mb4_0900_bin 逐字节判等。尤其 refresh_token_hash 上挂着唯一键， +// 大小写不敏感会让两个不同的哈希互相顶掉，也等于放宽一个 bearer 凭据的匹配条件。 +// +// ip 与 user_agent 只写不读（审计用，从不出现在任何 WHERE 里），显式排序规则对它们 +// 没有意义，留表默认即可——只在真正参与比较的列上写排序规则，读的人才知道哪些列的 +// 判等语义是被刻意选过的。 // // idx_dtokens_device_active 把 revoked_at 放进键里代替 PG 的 `WHERE revoked_at = 0`， // 理由同 devices：一条复合索引就能服务查询，不必为此加生成列。 @@ -22,14 +26,14 @@ func migration202605200004() *gormigrate.Migration { CREATE TABLE device_tokens ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, device_id bigint NOT NULL, - access_jti varchar(64) COLLATE utf8mb4_bin NOT NULL DEFAULT '', - refresh_token_hash varchar(64) COLLATE utf8mb4_bin NOT NULL, + access_jti varchar(64) COLLATE utf8mb4_0900_bin NOT NULL DEFAULT '', + refresh_token_hash varchar(64) COLLATE utf8mb4_0900_bin NOT NULL, refresh_expires_at bigint NOT NULL DEFAULT 0, last_used_at bigint NOT NULL DEFAULT 0, rotated_from_id bigint NOT NULL DEFAULT 0, revoked_at bigint NOT NULL DEFAULT 0, user_agent varchar(512) NOT NULL DEFAULT '', - ip varchar(45) COLLATE utf8mb4_bin, + ip varchar(45), createtime bigint NOT NULL DEFAULT 0, UNIQUE KEY uk_dtokens_refresh_hash (refresh_token_hash), KEY idx_dtokens_device_active (device_id, revoked_at) diff --git a/migrations/202605200005_device_flow_codes.go b/migrations/202605200005_device_flow_codes.go index 98d4bfbc..5db2d490 100644 --- a/migrations/202605200005_device_flow_codes.go +++ b/migrations/202605200005_device_flow_codes.go @@ -8,9 +8,15 @@ import ( // migration202605200005 创建 device_flow_codes 表（RFC 8628 的 device_code / // user_code 状态机）。 // -// device_code 与 user_code 都是凭据，用 utf8mb4_bin 逐字节判等：默认的 -// utf8mb4_0900_ai_ci 会让 device_code 的比较大小写不敏感，等于凭空放宽一个 bearer -// 凭据的匹配条件。两者的长度按生成器实际产出来定，不留无意义的余量—— +// device_code 是机器之间传递的 bearer 凭据，用 utf8mb4_0900_bin 逐字节判等：大小写 +// 不敏感等于凭空放宽一个凭据的匹配条件。 +// +// user_code 相反，它是印给人看、由人敲进浏览器的，所以用 utf8mb4_0900_as_ci +// 大小写不敏感——用户小写敲验证码必须也能对上。usercode.Normalize 已经会先转大写， +// 排序规则是第二层保障：将来多一条忘了 Normalize 的查询路径时，症状是「查不到这个 +// 验证码」这种很难联想到大小写的报错。 +// +// 两者的长度按生成器实际产出来定，不留无意义的余量—— // device_code 是 randomBase32(32)（32 字节 base32、无填充，恒为 52 位小写）， // user_code 是 usercode.Generate() 的 "XXX-XXX"（7 位大写）。device_code 还是主键， // InnoDB 会把主键塞进每一条二级索引，所以它的宽度是真实成本，不能随手写 varchar(255)。 @@ -28,10 +34,10 @@ func migration202605200005() *gormigrate.Migration { Migrate: func(tx *gorm.DB) error { return tx.Exec(` CREATE TABLE device_flow_codes ( - device_code varchar(64) COLLATE utf8mb4_bin PRIMARY KEY, - user_code varchar(16) COLLATE utf8mb4_bin NOT NULL, - device_kind varchar(32) COLLATE utf8mb4_bin NOT NULL, - client_fingerprint varchar(255) COLLATE utf8mb4_bin NOT NULL, + device_code varchar(64) COLLATE utf8mb4_0900_bin PRIMARY KEY, + user_code varchar(16) COLLATE utf8mb4_0900_as_ci NOT NULL, + device_kind varchar(32) COLLATE utf8mb4_0900_bin NOT NULL, + client_fingerprint varchar(255) COLLATE utf8mb4_0900_bin NOT NULL, platform varchar(64) NOT NULL DEFAULT '', version varchar(64) NOT NULL DEFAULT '', authorized_user_id bigint NOT NULL DEFAULT 0, diff --git a/migrations/202608090001_workspace_sync.go b/migrations/202608090001_workspace_sync.go index 3ad3857a..fb9e4bf7 100644 --- a/migrations/202608090001_workspace_sync.go +++ b/migrations/202608090001_workspace_sync.go @@ -13,9 +13,13 @@ import ( // 没有删除时间也没有冲突元数据。 // // sync_id / project_sync_id / agentred_fingerprint / kind 都是客户端自带的不透明 -// 标识，一律 utf8mb4_bin 逐字节判等。表默认的 utf8mb4_0900_ai_ci 大小写不敏感， -// 会让 "abc" 与 "ABC" 两个不同的 sync_id 撞上 uk_sync_objects_identity， -// 而且 `WHERE sync_id=?` 会取回另一行——同步的一切都建立在这个标识精确可比上。 +// 标识，一律 utf8mb4_0900_bin 逐字节判等，同步的一切都建立在这个标识精确可比上。 +// 两个坑都要躲开：表默认的 utf8mb4_0900_ai_ci 大小写不敏感，会让 "abc" 与 "ABC" +// 两个不同的 sync_id 撞上 uk_sync_objects_identity、`WHERE sync_id=?` 还会取回另一行； +// 而老的 utf8mb4_bin 虽然逐字节比较，却是 PAD SPACE，会忽略尾随空格，"x" 与 "x " +// 同样会互相顶掉。_0900_bin 才是既逐字节、又 NO PAD 的那一个。 +// +// agentred_fingerprint 要能和 devices.fingerprint 比较，两列排序规则必须一致。 // // uk_sync_objects_location 是 agentred 路径的账号内自然键，只约束存活的行：墓碑不占 // 自然键，否则删掉再建就建不回来。用 live_location_flag（存活时为 1、否则为 NULL） @@ -44,10 +48,10 @@ func migration202608090001() *gormigrate.Migration { CREATE TABLE sync_objects ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, user_id bigint NOT NULL, - kind varchar(32) COLLATE utf8mb4_bin NOT NULL, - sync_id varchar(255) COLLATE utf8mb4_bin NOT NULL, - project_sync_id varchar(255) COLLATE utf8mb4_bin NOT NULL DEFAULT '', - agentred_fingerprint varchar(255) COLLATE utf8mb4_bin NOT NULL DEFAULT '', + kind varchar(32) COLLATE utf8mb4_0900_bin NOT NULL, + sync_id varchar(255) COLLATE utf8mb4_0900_bin NOT NULL, + project_sync_id varchar(255) COLLATE utf8mb4_0900_bin NOT NULL DEFAULT '', + agentred_fingerprint varchar(255) COLLATE utf8mb4_0900_bin NOT NULL DEFAULT '', payload json NOT NULL DEFAULT ('{}'), version bigint NOT NULL, sync_updated_at bigint NOT NULL DEFAULT 0, @@ -78,7 +82,7 @@ func migration202608090001() *gormigrate.Migration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, ` CREATE TABLE sync_avatars ( user_id bigint NOT NULL, - content_hash varchar(64) COLLATE utf8mb4_bin NOT NULL, + content_hash varchar(64) COLLATE utf8mb4_0900_bin NOT NULL, content_type varchar(255) NOT NULL DEFAULT '', content mediumtext NOT NULL, byte_size bigint NOT NULL DEFAULT 0, @@ -88,7 +92,7 @@ func migration202608090001() *gormigrate.Migration { CREATE TABLE device_local_paths ( user_id bigint NOT NULL, device_id bigint NOT NULL, - project_sync_id varchar(255) COLLATE utf8mb4_bin NOT NULL, + project_sync_id varchar(255) COLLATE utf8mb4_0900_bin NOT NULL, path text NOT NULL, updatetime bigint NOT NULL DEFAULT 0, PRIMARY KEY (user_id, device_id, project_sync_id) diff --git a/migrations/202608100001_followed_sessions.go b/migrations/202608100001_followed_sessions.go index aba92a45..77cb4fc4 100644 --- a/migrations/202608100001_followed_sessions.go +++ b/migrations/202608100001_followed_sessions.go @@ -14,8 +14,9 @@ import ( // 在 MySQL 下发出 ON DUPLICATE KEY UPDATE 的自赋值形式），不新增行、 // 不重置首次关注时间；取消就是一条 DELETE，删不到也是成功。 // -// device_fingerprint 与 session_id 是目标设备与会话的不透明标识，用 utf8mb4_bin -// 逐字节判等：大小写不敏感会把两个不同的会话认成同一个，名单就指错了对象。 +// device_fingerprint 与 session_id 是目标设备与会话的不透明标识，用 +// utf8mb4_0900_bin 逐字节判等：大小写不敏感会把两个不同的会话认成同一个，名单就指错了 +// 对象。device_fingerprint 要能和 devices.fingerprint 比较，两列排序规则必须一致。 func migration202608100001() *gormigrate.Migration { return &gormigrate.Migration{ ID: "202608100001", @@ -24,8 +25,8 @@ func migration202608100001() *gormigrate.Migration { CREATE TABLE followed_sessions ( id bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, user_id bigint NOT NULL, - device_fingerprint varchar(255) COLLATE utf8mb4_bin NOT NULL, - session_id varchar(255) COLLATE utf8mb4_bin NOT NULL, + device_fingerprint varchar(255) COLLATE utf8mb4_0900_bin NOT NULL, + session_id varchar(255) COLLATE utf8mb4_0900_bin NOT NULL, followed_at bigint NOT NULL DEFAULT 0, createtime bigint NOT NULL DEFAULT 0, updatetime bigint NOT NULL DEFAULT 0, diff --git a/migrations/collation_test.go b/migrations/collation_test.go new file mode 100644 index 00000000..2fa7270f --- /dev/null +++ b/migrations/collation_test.go @@ -0,0 +1,107 @@ +package migrations + +import ( + "regexp" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +// ddlRecorder 是一个只记录、永远匹配的 sqlmock QueryMatcher：迁移的 DDL 不需要被 +// 逐句比对，需要的是把它们收集起来对整体做策略断言。 +type ddlRecorder struct{ seen *[]string } + +func (r ddlRecorder) Match(_, actualSQL string) error { + *r.seen = append(*r.seen, actualSQL) + return nil +} + +// captureMigrationDDL 直接调用每个迁移的 Migrate，收集它发出的全部 SQL。 +// +// 不走 RunMigrations：那会把 gormigrate 自己的建表/记账语句也混进来，而这里要断言的 +// 只是我们写的 DDL。 +func captureMigrationDDL(t *testing.T) string { + t.Helper() + var seen []string + sqlDB, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(ddlRecorder{seen: &seen})) + require.NoError(t, err) + defer func() { _ = sqlDB.Close() }() + + gormDB, err := gorm.Open(mysql.New(mysql.Config{ + Conn: sqlDB, SkipInitializeWithVersion: true, + }), &gorm.Config{}) + require.NoError(t, err) + + for range 64 { + mock.ExpectExec(".*").WillReturnResult(sqlmock.NewResult(0, 0)) + } + for _, m := range migrationList() { + require.NoError(t, m.Migrate(gormDB), "migration %s", m.ID) + } + require.NotEmpty(t, seen) + return strings.Join(seen, "\n") +} + +// 排序规则必须显式选 NO PAD 的那一族。utf8mb4_bin 是 **PAD SPACE**：在它下面 +// 'x' 与 'x ' 比较相等，于是尾随空格不同的两个 sync_id 会撞上唯一键、 +// `WHERE sync_id=?` 也会取回另一行。PG 的 text 是逐字节比较、尾随空格显著， +// utf8mb4_0900_bin 才是与之等价的那个。这个差别在任何单测和肉眼 review 里都看不出来， +// 只能靠这条断言挡住。 +func TestMigrationDDL_NeverUsesPadSpaceCollation(t *testing.T) { + ddl := captureMigrationDDL(t) + + // utf8mb4_bin 后面不能紧跟别的字符（否则会把 utf8mb4_0900_bin 也匹配掉）。 + padSpace := regexp.MustCompile(`utf8mb4_bin\b`) + assert.Empty(t, padSpace.FindAllString(ddl, -1), + "utf8mb4_bin 是 PAD SPACE，会忽略尾随空格；标识列要用 utf8mb4_0900_bin") +} + +// 每张表都要钉住引擎与字符集，否则 schema 会随服务器的 character_set_server 变化—— +// 同一份迁移在两台配置不同的 MySQL 上会建出语义不同的表。 +func TestMigrationDDL_PinsEngineAndCharsetOnEveryTable(t *testing.T) { + ddl := captureMigrationDDL(t) + + creates := regexp.MustCompile(`(?i)CREATE TABLE\s+(\w+)`).FindAllStringSubmatch(ddl, -1) + pinned := regexp.MustCompile(`ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`) + assert.Len(t, pinned.FindAllString(ddl, -1), len(creates), + "CREATE TABLE 的数量与钉住 ENGINE/CHARSET 的数量必须相等") +} + +// 人手输入的标识符按大小写不敏感判等，机器生成的不透明标识按逐字节判等。 +// +// email 与 user_code 是人打出来的：同一个人用 "A@b.C" 和 "a@b.c" 注册必须是同一个账号， +// 用小写敲验证码也必须能对上。但用的是 as_ci 而不是表默认的 ai_ci——ai_ci 连重音都折叠， +// 会把 e@x.c 和 é@x.c 当成同一个邮箱，那是两个不同的收件人。 +func TestMigrationDDL_HumanEnteredIdentifiersAreCaseInsensitive(t *testing.T) { + ddl := captureMigrationDDL(t) + + for _, want := range []string{ + "email varchar(320) COLLATE utf8mb4_0900_as_ci NOT NULL", // users + "email varchar(320) COLLATE utf8mb4_0900_as_ci NOT NULL,", // user_identities + "user_code varchar(16) COLLATE utf8mb4_0900_as_ci NOT NULL", + } { + assert.Contains(t, ddl, want) + } +} + +// 不透明标识（客户端/服务端生成的同步标识、指纹、凭据、哈希）必须逐字节判等： +// 大小写或尾随空格不同的两个值是两个不同的东西，折叠它们会让同步张冠李戴、 +// 也会凭空放宽一个 bearer 凭据的匹配条件。 +func TestMigrationDDL_OpaqueIdentifiersAreByteExact(t *testing.T) { + ddl := captureMigrationDDL(t) + + for _, col := range []string{ + "sync_id", "project_sync_id", "agentred_fingerprint", + "fingerprint", "client_fingerprint", "device_fingerprint", "session_id", + "device_code", "refresh_token_hash", "access_jti", "content_hash", + "provider_uid", + } { + pattern := regexp.MustCompile(`\b` + col + `\s+varchar\(\d+\) COLLATE utf8mb4_0900_bin\b`) + assert.Regexp(t, pattern, ddl, "%s 必须是 utf8mb4_0900_bin", col) + } +}