fix: hot-load file changes on every sync in long-running services

README.md

@@ -39,6 +39,12 @@ baton-file -i data.csv
 baton-file -i data.xlsx
 ```
 
+## Hot-Load (File Change Detection)
+
+When baton-file runs as a long-lived service, edits to the input file are picked up automatically at the start of the next sync cycle — no restart required. This applies to the **data** sections: users, resources, entitlements, grants, and inheritance mappings. **Schema changes** — introducing a brand-new resource type, or changing an existing type's trait — still require a service restart, because the SDK registers resource types and their traits once at process startup. If the edited file fails validation, the sync fails with the validation error and the connector keeps serving the last successfully loaded data. If the file changes while a sync is in flight (for example via health-check revalidation), affected listings restart safely against the new contents instead of resuming into stale page offsets; if the same listing restarts more than three times in a row — with no page served under an unchanged file in between — the sync fails with a "rewritten faster than syncs can read it" error and the next sync starts fresh. A file that cannot even be read consistently (it keeps changing during the load itself) fails the sync with an "input file kept changing while being loaded" error, and likewise recovers on the next sync once the file is stable. A file that is valid but has no data rows is a legitimate empty state: the connector registers its standard resource types (user, group, role, app, secret), syncs empty, and hot-loads data rows as they are added — though rows using custom resource type IDs still require a restart, like any schema change. Hot-load does require a service that started successfully: if the input file is **invalid** when the service first starts, no resource types get registered and syncs fail until the file is fixed **and the service is restarted**.
+
+**IMPORTANT (maintainers and AI agents):** this behavior is a deliberate contract, not an accident of implementation. The SDK calls `ResourceSyncers()` exactly once per process, so `Validate()` — which runs at the start of every sync — is the only per-sync hook and is where the file is re-read and the shared cache is republished (see `cacheHolder` in `pkg/connector/connector.go`). The construction-time load in `ResourceSyncers()` is the *first* load, never the only one: do not capture cache snapshots in builders, and do not assume the SDK refreshes data between syncs (it does not — that assumption caused a regression once already). `TestHotReload_DataChangesPickedUpBySync` in `pkg/connector/hot_reload_test.go` enforces this contract; if it fails after your change, the change is wrong, not the test.
+
 ## Templates
 
 Full templates demonstrate every field and feature. Quickstart templates have the minimum to get a working sync (two users, a group, a role, and direct grants).

docs/connector.mdx

@@ -34,6 +34,7 @@ Resource types are defined by the input file (via each row's `trait`), so the ex
 - **Read-only.** The connector cannot create, modify, revoke, or deprovision access in the source system — it only reads what the file describes.
 - **Full sync only.** Every sync reads the entire file; there is no incremental/targeted sync.
 - **File-defined data.** C1 only knows what the file contains. Resource IDs must be globally unique across users and resources.
+- **File edits apply on the next sync.** When the connector runs as a long-lived service, changes to the file's data — users, resources, entitlements, grants, and inheritance mappings — are picked up automatically at the start of the next sync, with no restart needed. Adding a new resource type, or changing an existing type's trait, does require restarting the connector. If an edited file fails validation, the sync fails with the validation error and the connector keeps serving the last successfully loaded data. A file with no data rows is valid: the connector syncs empty and picks up rows as they are added. If the file is rewritten repeatedly while a sync is running, the sync may fail with a "rewritten faster than syncs can read it" or "kept changing while being loaded" error — the next sync starts fresh; avoid continuous rewrites during syncs. If the file is invalid when the connector first starts (rather than edited mid-run), fix the file and restart the connector.
 
 ## Gather File connector credentials
 

pkg/connector/external_grants_test.go

@@ -50,7 +50,7 @@ func TestGrants_ExternalGrant_AttributeMatch(t *testing.T) {
 	cache, err := newSyncCache(ctx, data)
 	require.NoError(t, err)
 
-	b := &resourceBuilder{cache: cache, resourceType: cache.resourceTypes["app"]}
+	b := testBuilder(cache, cache.resourceTypes["app"])
 	grants := allGrants(t, b, cache.resources["payroll-app"])
 	require.Len(t, grants, 1)
 
@@ -77,7 +77,7 @@ func TestGrants_ExternalGrant_MatchAll(t *testing.T) {
 	cache, err := newSyncCache(ctx, data)
 	require.NoError(t, err)
 
-	b := &resourceBuilder{cache: cache, resourceType: cache.resourceTypes["app"]}
+	b := testBuilder(cache, cache.resourceTypes["app"])
 	grants := allGrants(t, b, cache.resources["payroll-app"])
 	require.Len(t, grants, 1)
 
@@ -100,7 +100,7 @@ func TestGrants_ExternalGrant_MatchID(t *testing.T) {
 	cache, err := newSyncCache(ctx, data)
 	require.NoError(t, err)
 
-	b := &resourceBuilder{cache: cache, resourceType: cache.resourceTypes["app"]}
+	b := testBuilder(cache, cache.resourceTypes["app"])
 	grants := allGrants(t, b, cache.resources["payroll-app"])
 	require.Len(t, grants, 1)
 
@@ -125,7 +125,7 @@ func TestGrants_ExternalGrant_Expansion(t *testing.T) {
 	cache, err := newSyncCache(ctx, data)
 	require.NoError(t, err)
 
-	b := &resourceBuilder{cache: cache, resourceType: cache.resourceTypes["app"]}
+	b := testBuilder(cache, cache.resourceTypes["app"])
 	grants := allGrants(t, b, cache.resources["payroll-app"])
 	require.Len(t, grants, 1)
 
@@ -156,7 +156,7 @@ func TestGrants_ExternalGrant_Expansion_DefaultDepthIsFull(t *testing.T) {
 	cache, err := newSyncCache(ctx, data)
 	require.NoError(t, err)
 
-	b := &resourceBuilder{cache: cache, resourceType: cache.resourceTypes["app"]}
+	b := testBuilder(cache, cache.resourceTypes["app"])
 	grants := allGrants(t, b, cache.resources["payroll-app"])
 	require.Len(t, grants, 1)
 
@@ -190,7 +190,7 @@ func TestGrants_ExternalGrant_Expansion_SkipsInvalidCombos(t *testing.T) {
 	cache, err := newSyncCache(ctx, data)
 	require.NoError(t, err)
 
-	b := &resourceBuilder{cache: cache, resourceType: cache.resourceTypes["app"]}
+	b := testBuilder(cache, cache.resourceTypes["app"])
 	grants := allGrants(t, b, cache.resources["payroll-app"])
 	require.Empty(t, grants, "invalid expansion combinations must be skipped")
 }
@@ -232,7 +232,7 @@ func TestGrants_ExternalGrant_SkipsInvalidRows(t *testing.T) {
 	cache, err := newSyncCache(ctx, data)
 	require.NoError(t, err)
 
-	b := &resourceBuilder{cache: cache, resourceType: cache.resourceTypes["app"]}
+	b := testBuilder(cache, cache.resourceTypes["app"])
 	grants := allGrants(t, b, cache.resources["payroll-app"])
 	require.Empty(t, grants, "invalid external grant rows must be skipped, not emitted")
 }