Skip to content

Manual Migrations #5467

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
gefjon wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Manual Migrations #5467

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d7eb1ec
moduledef extensions for manual migrations
gefjon Jun 30, 2026
fba3483
Re-organize schema crate migration defs
gefjon Jul 1, 2026
a5bd46a
Scaffold `ponder_manual_migrate` and re-work call chain that reaches it
gefjon Jul 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion crates/client-api/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use spacetimedb_client_api_messages::http::{SqlStmtResult, SqlStmtStats};
use spacetimedb_client_api_messages::name::{DomainName, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld};
use spacetimedb_lib::{ProductTypeElement, ProductValue};
use spacetimedb_paths::server::ModuleLogsDir;
use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle};
use spacetimedb_schema::migrate::{MigrationPolicy, PrettyPrintStyle};
use thiserror::Error;
use tokio::sync::watch;

Expand Down
6 changes: 3 additions & 3 deletions crates/client-api/src/routes/database.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10;
use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9;
use spacetimedb_lib::http as st_http;
use spacetimedb_lib::{sats, AlgebraicValue, Hash, ProductValue, Timestamp};
use spacetimedb_schema::auto_migrate::{
use spacetimedb_schema::migrate::{
MigrationPolicy as SchemaMigrationPolicy, MigrationToken, PrettyPrintStyle as AutoMigratePrettyPrintStyle,
};
use tokio::sync::oneshot;
Expand Down Expand Up @@ -1060,7 +1060,7 @@ pub async fn publish<S: NodeDelegate + ControlStateDelegate + Authorization>(
})
};
match maybe_updated {
Some(UpdateDatabaseResult::AutoMigrateError(errs)) => {
Some(UpdateDatabaseResult::MigrationPlanningError(errs)) => {
Err(bad_request(format!("Database update rejected: {errs}").into()))
}
Some(UpdateDatabaseResult::ErrorExecutingMigration(err)) => Err(bad_request(
Expand Down Expand Up @@ -1282,7 +1282,7 @@ pub async fn pre_publish<S: NodeDelegate + ControlStateDelegate + Authorization>
major_version_upgrade,
}))
}
MigratePlanResult::AutoMigrationError {
MigratePlanResult::PlanningError {
error: e,
major_version_upgrade,
} => {
Expand Down
9 changes: 4 additions & 5 deletions crates/core/src/host/host_controller.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ use log::{info, trace, warn};
use parking_lot::Mutex;
use scopeguard::defer;
use spacetimedb_commitlog::SizeOnDisk;
use spacetimedb_data_structures::error_stream::ErrorStream;
use spacetimedb_data_structures::map::{IntMap, IntSet};
use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS;
use spacetimedb_datastore::db_metrics::DB_METRICS;
Expand All @@ -41,8 +40,8 @@ use spacetimedb_lib::{identity::AuthCtx, AlgebraicValue, Identity, Timestamp};
use spacetimedb_paths::server::{ModuleLogsDir, ServerDataDir};
use spacetimedb_runtime::AbortHandle;
use spacetimedb_sats::hash::Hash;
use spacetimedb_schema::auto_migrate::{ponder_migrate, AutoMigrateError, MigrationPolicy, PrettyPrintStyle};
use spacetimedb_schema::def::{ModuleDef, RawModuleDefVersion};
use spacetimedb_schema::migrate::{ponder_migrate, MigrationPolicy, PonderMigrateError, PrettyPrintStyle};
use spacetimedb_table::page_pool::PagePool;
use std::future::Future;
use std::ops::Deref;
Expand Down Expand Up @@ -1444,7 +1443,7 @@ impl Host {
plan: plan.pretty_print(style)?.into(),
major_version_upgrade,
},
Err(e) => MigratePlanResult::AutoMigrationError {
Err(e) => MigratePlanResult::PlanningError {
error: e,
major_version_upgrade,
},
Expand Down Expand Up @@ -1474,8 +1473,8 @@ pub enum MigratePlanResult {
breaks_client: bool,
major_version_upgrade: bool,
},
AutoMigrationError {
error: ErrorStream<AutoMigrateError>,
PlanningError {
error: PonderMigrateError,
major_version_upgrade: bool,
},
}
Expand Down
5 changes: 2 additions & 3 deletions crates/core/src/host/module_host.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ use spacetimedb_client_api_messages::energy::FunctionBudget;
use spacetimedb_client_api_messages::websocket::common::{ByteListLen as _, RowListLen as _};
use spacetimedb_client_api_messages::websocket::v1::{self as ws_v1};
use spacetimedb_client_api_messages::websocket::v2::{self as ws_v2};
use spacetimedb_data_structures::error_stream::ErrorStream;
use spacetimedb_data_structures::map::{HashCollectionExt as _, HashSet};
use spacetimedb_datastore::error::DatastoreError;
use spacetimedb_datastore::execution_context::{Workload, WorkloadType};
Expand All @@ -64,9 +63,9 @@ use spacetimedb_primitives::{HttpHandlerId, ProcedureId, TableId, ViewFnPtr, Vie
use spacetimedb_query::compile_subscription;
use spacetimedb_sats::raw_identifier::RawIdentifier;
use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductValue};
use spacetimedb_schema::auto_migrate::{AutoMigrateError, MigrationPolicy};
use spacetimedb_schema::def::{ModuleDef, ProcedureDef, ReducerDef, ViewDef};
use spacetimedb_schema::identifier::Identifier;
use spacetimedb_schema::migrate::{MigrationPolicy, PonderMigrateError};
use spacetimedb_schema::reducer_name::ReducerName;
use spacetimedb_schema::table_name::TableName;
use std::collections::VecDeque;
Expand Down Expand Up @@ -1477,7 +1476,7 @@ pub enum UpdateDatabaseResult {
/// `None` if the database is in-memory only.
durable_offset: Option<DurableOffset>,
},
AutoMigrateError(Box<ErrorStream<AutoMigrateError>>),
MigrationPlanningError(Box<PonderMigrateError>),
ErrorExecutingMigration(anyhow::Error),
}
impl UpdateDatabaseResult {
Expand Down
2 changes: 1 addition & 1 deletion crates/core/src/host/v8/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,9 +102,9 @@ use spacetimedb_client_api_messages::energy::FunctionBudget;
use spacetimedb_datastore::locking_tx_datastore::FuncCallType;
use spacetimedb_datastore::traits::Program;
use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp};
use spacetimedb_schema::auto_migrate::MigrationPolicy;
use spacetimedb_schema::def::ModuleDef;
use spacetimedb_schema::identifier::Identifier;
use spacetimedb_schema::migrate::MigrationPolicy;
use spacetimedb_table::static_assert_size;
use std::cell::Cell;
use std::num::NonZeroUsize;
Expand Down
6 changes: 4 additions & 2 deletions crates/core/src/host/wasm_common/module_host_actor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ use spacetimedb_lib::{bsatn, http as st_http, ConnectionId, Hash, ProductType, R
use spacetimedb_primitives::{HttpHandlerId, ProcedureId, TableId, ViewFnPtr, ViewId};
use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type;
use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, Deserialize, ProductValue, Typespace, WithTypespace};
use spacetimedb_schema::auto_migrate::{MigratePlan, MigrationPolicy, MigrationPolicyError};
use spacetimedb_schema::def::deserialize::FunctionDef;
use spacetimedb_schema::def::{ModuleDef, ViewDef};
use spacetimedb_schema::identifier::Identifier;
use spacetimedb_schema::migrate::{MigratePlan, MigrationPolicy, MigrationPolicyError};
use spacetimedb_schema::reducer_name::ReducerName;
use spacetimedb_subscription::SubscriptionPlan;
use std::sync::Arc;
Expand Down Expand Up @@ -661,7 +661,9 @@ impl InstanceCommon {
Ok(plan) => plan,
Err(e) => {
return match e {
MigrationPolicyError::AutoMigrateFailure(e) => Ok(UpdateDatabaseResult::AutoMigrateError(e.into())),
MigrationPolicyError::PlanningFailure(e) => {
Ok(UpdateDatabaseResult::MigrationPlanningError(e.into()))
}
_ => Ok(UpdateDatabaseResult::ErrorExecutingMigration(e.into())),
}
}
Expand Down
46 changes: 23 additions & 23 deletions crates/engine/src/update.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ use spacetimedb_lib::db::auth::StTableType;
use spacetimedb_lib::identity::AuthCtx;
use spacetimedb_lib::AlgebraicValue;
use spacetimedb_primitives::{ColSet, TableId};
use spacetimedb_schema::auto_migrate::{AutoMigratePlan, AutoMigrateStep, ManualMigratePlan, MigratePlan};
use spacetimedb_schema::def::{ModuleDef, ModuleDefLookup, TableDef, ViewDef};
use spacetimedb_schema::migrate::{AutoMigratePlan, AutoMigrateStep, ManualMigratePlan, MigratePlan};
use spacetimedb_schema::schema::{
column_schemas_from_defs, ConstraintSchema, IndexSchema, Schema, SequenceSchema, TableSchema,
};
Expand Down Expand Up @@ -185,7 +185,7 @@ fn auto_migrate_database(

for precheck in plan.prechecks {
match precheck {
spacetimedb_schema::auto_migrate::AutoMigratePrecheck::CheckAddSequenceRangeValid(sequence_name) => {
spacetimedb_schema::migrate::AutoMigratePrecheck::CheckAddSequenceRangeValid(sequence_name) => {
let table_def = plan.new.stored_in_table_def(sequence_name).unwrap();
let sequence_def = &table_def.sequences[sequence_name];
let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap();
Expand Down Expand Up @@ -225,7 +225,7 @@ fn auto_migrate_database(

for step in plan.steps {
match step {
spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveTable(table_name) => {
AutoMigrateStep::RemoveTable(table_name) => {
let table_id = stdb.table_id_from_name_mut(tx, table_name)?.unwrap();

if stdb.table_row_count_mut(tx, table_id).unwrap_or(0) > 0 {
Expand All @@ -238,7 +238,7 @@ fn auto_migrate_database(
log!(logger, "Dropping table `{table_name}`");
stdb.drop_table(tx, table_id)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::AddTable(table_name) => {
AutoMigrateStep::AddTable(table_name) => {
let table_def: &TableDef = plan.new.expect_lookup(table_name);

// Recursively sets IDs to 0.
Expand All @@ -249,22 +249,22 @@ fn auto_migrate_database(

stdb.create_table(tx, table_schema)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::AddView(view_name) => {
AutoMigrateStep::AddView(view_name) => {
let view_def: &ViewDef = plan.new.expect_lookup(view_name);
stdb.create_view(tx, plan.new, view_def)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveView(view_name) => {
AutoMigrateStep::RemoveView(view_name) => {
let view_id = stdb.view_id_from_name_mut(tx, view_name)?.unwrap();
stdb.drop_view(tx, view_id)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::UpdateView(_) => {
AutoMigrateStep::UpdateView(_) => {
// if we already have to disconnect clients, no need to set
// `EvaluateSubscribedViews` as clients will be disconnected anyway
if !matches!(res, UpdateResult::RequiresClientDisconnect) {
res = UpdateResult::EvaluateSubscribedViews;
}
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::AddIndex(index_name) => {
AutoMigrateStep::AddIndex(index_name) => {
let table_def = plan.new.stored_in_table_def(index_name).unwrap();
let index_def = table_def.indexes.get(index_name).unwrap();
let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap();
Expand All @@ -283,7 +283,7 @@ fn auto_migrate_database(

stdb.create_index(tx, index_schema, is_unique)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveIndex(index_name) => {
AutoMigrateStep::RemoveIndex(index_name) => {
let table_def = plan.old.stored_in_table_def(index_name).unwrap();

let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap();
Expand All @@ -298,7 +298,7 @@ fn auto_migrate_database(
log!(logger, "Dropping index `{}` on table `{}`", index_name, table_def.name);
stdb.drop_index(tx, index_schema.index_id)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveConstraint(constraint_name) => {
AutoMigrateStep::RemoveConstraint(constraint_name) => {
let table_def = plan.old.stored_in_table_def(constraint_name).unwrap();

let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap();
Expand All @@ -317,7 +317,7 @@ fn auto_migrate_database(
);
stdb.drop_constraint(tx, constraint_schema.constraint_id)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::AddConstraint(constraint_name) => {
AutoMigrateStep::AddConstraint(constraint_name) => {
let table_def = plan
.new
.stored_in_table_def(constraint_name)
Expand All @@ -340,7 +340,7 @@ fn auto_migrate_database(
);
stdb.create_constraint(tx, constraint_schema)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::AddSequence(sequence_name) => {
AutoMigrateStep::AddSequence(sequence_name) => {
let table_def = plan.new.stored_in_table_def(sequence_name).unwrap();
let sequence_def = table_def.sequences.get(sequence_name).unwrap();

Expand All @@ -357,7 +357,7 @@ fn auto_migrate_database(
SequenceSchema::from_module_def(plan.new, sequence_def, table_schema.table_id, 0.into());
stdb.create_sequence(tx, sequence_schema)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveSequence(sequence_name) => {
AutoMigrateStep::RemoveSequence(sequence_name) => {
let table_def = plan.old.stored_in_table_def(sequence_name).unwrap();

let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap();
Expand All @@ -376,7 +376,7 @@ fn auto_migrate_database(
);
stdb.drop_sequence(tx, sequence_schema.sequence_id)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeColumns(table_name) => {
AutoMigrateStep::ChangeColumns(table_name) => {
let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap();
let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap();
let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id);
Expand All @@ -385,7 +385,7 @@ fn auto_migrate_database(

stdb.alter_table_row_type(tx, table_id, column_schemas)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::ReschemaEventTable(table_name) => {
AutoMigrateStep::ReschemaEventTable(table_name) => {
let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap();
let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap();
let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id);
Expand All @@ -394,33 +394,33 @@ fn auto_migrate_database(

stdb.alter_event_table_row_type(tx, table_id, column_schemas)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeAccess(table_name) => {
AutoMigrateStep::ChangeAccess(table_name) => {
let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap();
stdb.alter_table_access(tx, table_name, table_def.table_access.into())?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangePrimaryKey(table_name) => {
AutoMigrateStep::ChangePrimaryKey(table_name) => {
let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap();
log!(logger, "Changing primary key for table `{table_name}`");
stdb.alter_table_primary_key(tx, table_name, table_def.primary_key)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::AddSchedule(_) => {
AutoMigrateStep::AddSchedule(_) => {
anyhow::bail!("Adding schedules is not yet implemented");
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveSchedule(_) => {
AutoMigrateStep::RemoveSchedule(_) => {
anyhow::bail!("Removing schedules is not yet implemented");
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::AddRowLevelSecurity(sql_rls) => {
AutoMigrateStep::AddRowLevelSecurity(sql_rls) => {
log!(logger, "Adding row-level security `{sql_rls}`");
let rls = plan.new.lookup_expect(sql_rls);
let rls = RowLevelExpr::build_row_level_expr(tx, &auth_ctx, rls)?;

stdb.create_row_level_security(tx, rls.def)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveRowLevelSecurity(sql_rls) => {
AutoMigrateStep::RemoveRowLevelSecurity(sql_rls) => {
log!(logger, "Removing-row level security `{sql_rls}`");
stdb.drop_row_level_security(tx, sql_rls.clone())?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::AddColumns(table_name) => {
AutoMigrateStep::AddColumns(table_name) => {
let table_def = plan
.new
.stored_in_table_def(&table_name.clone().into())
Expand All @@ -435,7 +435,7 @@ fn auto_migrate_database(
.collect();
stdb.add_columns_to_table_mut_tx(tx, table_id, column_schemas, default_values)?;
}
spacetimedb_schema::auto_migrate::AutoMigrateStep::DisconnectAllUsers => {
AutoMigrateStep::DisconnectAllUsers => {
log!(logger, "Disconnecting all users");
// It does not disconnect clients right away,
// but send response indicated that caller should drop clients
Expand Down
Loading
Loading