Skip to content

Commit 847de8c

Browse files
fix(state_sync): skip interval removal when there is nothing to remove (#6027)
Signed-off-by: Amir Vakili <AVakili@Voleon.com> Co-authored-by: Cortland Goffena <30168413+cmgoffena13@users.noreply.github.com>
1 parent c0ae2db commit 847de8c

3 files changed

Lines changed: 75 additions & 0 deletions

File tree

‎sqlmesh/core/state_sync/db/interval.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,9 @@ def remove_intervals(
108108
for snapshot in all_snapshots
109109
]
110110

111+
if not intervals_to_remove:
112+
return
113+
111114
if logger.isEnabledFor(logging.INFO):
112115
snapshot_ids = ", ".join(str(s.snapshot_id) for s, _ in intervals_to_remove)
113116
logger.info("Removing interval for snapshots: %s", snapshot_ids)

‎tests/core/integration/test_restatement.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,44 @@ def test_restatement_plan_ignores_changes(init_and_plan_context: t.Callable):
8383
context.apply(plan)
8484

8585

86+
@time_machine.travel("2023-01-08 15:00:00 UTC")
87+
def test_prod_restatement_with_unbackfilled_dev_version(init_and_plan_context: t.Callable):
88+
"""
89+
Scenario:
90+
Prod is built. A breaking change is planned to `dev` with `--skip-backfill`,
91+
so `dev` holds a different snapshot version with no interval rows. Prod is
92+
then restated.
93+
Outcome:
94+
RestatementStage tries to clear `dev` intervals for that other version, finds
95+
none in `_intervals`, and no-ops instead of crashing on an empty insert. Prod
96+
restatement still applies; the un-backfilled `dev` snapshot still has no
97+
intervals.
98+
"""
99+
context, plan = init_and_plan_context("examples/sushi")
100+
context.apply(plan)
101+
102+
prod_snapshot_id = context.get_snapshot("sushi.waiter_revenue_by_day").snapshot_id
103+
104+
context.upsert_model(
105+
add_projection_to_model(t.cast(SqlModel, context.get_model("sushi.waiter_revenue_by_day")))
106+
)
107+
context.plan("dev", skip_backfill=True, auto_apply=True, no_prompts=True)
108+
109+
dev_snapshot_id = context.get_snapshot("sushi.waiter_revenue_by_day").snapshot_id
110+
assert dev_snapshot_id != prod_snapshot_id
111+
assert not context.state_sync.get_snapshots([dev_snapshot_id])[dev_snapshot_id].intervals
112+
113+
context.plan(
114+
restate_models=["sushi.waiter_revenue_by_day"],
115+
start="2023-01-07",
116+
end="2023-01-08",
117+
auto_apply=True,
118+
no_prompts=True,
119+
)
120+
121+
assert not context.state_sync.get_snapshots([dev_snapshot_id])[dev_snapshot_id].intervals
122+
123+
86124
@time_machine.travel("2023-01-08 15:00:00 UTC")
87125
def test_restatement_plan_across_environments_snapshot_with_shared_version(
88126
init_and_plan_context: t.Callable,

‎tests/core/state_sync/test_state_sync.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,40 @@ def test_remove_interval_missing_snapshot(
400400
]
401401

402402

403+
def test_remove_interval_no_matching_intervals(
404+
state_sync: EngineAdapterStateSync, make_snapshot: t.Callable
405+
) -> None:
406+
snapshot = make_snapshot(
407+
SqlModel(
408+
name="a",
409+
cron="@daily",
410+
query=parse_one("select 1, ds"),
411+
),
412+
version="a",
413+
)
414+
state_sync.push_snapshots([snapshot])
415+
416+
# The snapshot has never been backfilled, so there are no rows to expand the shared versions from
417+
state_sync.remove_intervals(
418+
[(snapshot, snapshot.inclusive_exclusive("2020-01-15", "2020-01-17"))],
419+
remove_shared_versions=True,
420+
)
421+
422+
remove_records_count = state_sync.engine_adapter.fetchone(
423+
"SELECT COUNT(*) FROM sqlmesh._intervals WHERE name = '\"a\"' AND version = 'a' AND is_removed"
424+
)[0] # type: ignore
425+
assert remove_records_count == 0
426+
427+
assert not state_sync.get_snapshots([snapshot])[snapshot.snapshot_id].intervals
428+
429+
430+
def test_remove_interval_empty_input(state_sync: EngineAdapterStateSync) -> None:
431+
state_sync.remove_intervals([])
432+
state_sync.remove_intervals([], remove_shared_versions=True)
433+
434+
assert state_sync.engine_adapter.fetchone("SELECT COUNT(*) FROM sqlmesh._intervals")[0] == 0 # type: ignore
435+
436+
403437
def test_refresh_snapshot_intervals(
404438
state_sync: EngineAdapterStateSync, make_snapshot: t.Callable
405439
) -> None:

0 commit comments

Comments
 (0)