diff --git a/Cargo.lock b/Cargo.lock index 4e224012f03..d54e70ca7a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8737,6 +8737,7 @@ name = "spacetimedb-schema" version = "2.7.0" dependencies = [ "anyhow", + "blake3", "convert_case 0.6.0", "derive_more 0.99.20", "enum-as-inner", diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 7fd6f238128..4dcd4cea7d8 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -19,7 +19,7 @@ use spacetimedb_client_api_messages::http::{SqlStmtResult, SqlStmtStats}; use spacetimedb_client_api_messages::name::{DomainName, InsertDomainResult, RegisterTldResult, SetDomainsResult, Tld}; use spacetimedb_lib::{ProductTypeElement, ProductValue}; use spacetimedb_paths::server::ModuleLogsDir; -use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; +use spacetimedb_schema::migrate::{MigrationPolicy, PrettyPrintStyle}; use thiserror::Error; use tokio::sync::watch; diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 85698f02a52..bd561c88c03 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -45,7 +45,7 @@ use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::http as st_http; use spacetimedb_lib::{sats, AlgebraicValue, Hash, ProductValue, Timestamp}; -use spacetimedb_schema::auto_migrate::{ +use spacetimedb_schema::migrate::{ MigrationPolicy as SchemaMigrationPolicy, MigrationToken, PrettyPrintStyle as AutoMigratePrettyPrintStyle, }; use tokio::sync::oneshot; @@ -1060,7 +1060,7 @@ pub async fn publish( }) }; match maybe_updated { - Some(UpdateDatabaseResult::AutoMigrateError(errs)) => { + Some(UpdateDatabaseResult::MigrationPlanningError(errs)) => { Err(bad_request(format!("Database update rejected: {errs}").into())) } Some(UpdateDatabaseResult::ErrorExecutingMigration(err)) => Err(bad_request( @@ -1282,7 +1282,7 @@ pub async fn pre_publish major_version_upgrade, })) } - MigratePlanResult::AutoMigrationError { + MigratePlanResult::PlanningError { error: e, major_version_upgrade, } => { diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index dc34677b4a1..0230111218f 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -29,7 +29,6 @@ use log::{info, trace, warn}; use parking_lot::Mutex; use scopeguard::defer; use spacetimedb_commitlog::SizeOnDisk; -use spacetimedb_data_structures::error_stream::ErrorStream; use spacetimedb_data_structures::map::{IntMap, IntSet}; use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS; use spacetimedb_datastore::db_metrics::DB_METRICS; @@ -41,8 +40,8 @@ use spacetimedb_lib::{identity::AuthCtx, AlgebraicValue, Identity, Timestamp}; use spacetimedb_paths::server::{ModuleLogsDir, ServerDataDir}; use spacetimedb_runtime::AbortHandle; use spacetimedb_sats::hash::Hash; -use spacetimedb_schema::auto_migrate::{ponder_migrate, AutoMigrateError, MigrationPolicy, PrettyPrintStyle}; use spacetimedb_schema::def::{ModuleDef, RawModuleDefVersion}; +use spacetimedb_schema::migrate::{ponder_migrate, MigrationPolicy, PonderMigrateError, PrettyPrintStyle}; use spacetimedb_table::page_pool::PagePool; use std::future::Future; use std::ops::Deref; @@ -1444,7 +1443,7 @@ impl Host { plan: plan.pretty_print(style)?.into(), major_version_upgrade, }, - Err(e) => MigratePlanResult::AutoMigrationError { + Err(e) => MigratePlanResult::PlanningError { error: e, major_version_upgrade, }, @@ -1474,8 +1473,8 @@ pub enum MigratePlanResult { breaks_client: bool, major_version_upgrade: bool, }, - AutoMigrationError { - error: ErrorStream, + PlanningError { + error: PonderMigrateError, major_version_upgrade: bool, }, } diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 8a1bbc6b414..c534cd8a7a1 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -43,7 +43,6 @@ use spacetimedb_client_api_messages::energy::FunctionBudget; use spacetimedb_client_api_messages::websocket::common::{ByteListLen as _, RowListLen as _}; use spacetimedb_client_api_messages::websocket::v1::{self as ws_v1}; use spacetimedb_client_api_messages::websocket::v2::{self as ws_v2}; -use spacetimedb_data_structures::error_stream::ErrorStream; use spacetimedb_data_structures::map::{HashCollectionExt as _, HashSet}; use spacetimedb_datastore::error::DatastoreError; use spacetimedb_datastore::execution_context::{Workload, WorkloadType}; @@ -64,9 +63,9 @@ use spacetimedb_primitives::{HttpHandlerId, ProcedureId, TableId, ViewFnPtr, Vie use spacetimedb_query::compile_subscription; use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductValue}; -use spacetimedb_schema::auto_migrate::{AutoMigrateError, MigrationPolicy}; use spacetimedb_schema::def::{ModuleDef, ProcedureDef, ReducerDef, ViewDef}; use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::migrate::{MigrationPolicy, PonderMigrateError}; use spacetimedb_schema::reducer_name::ReducerName; use spacetimedb_schema::table_name::TableName; use std::collections::VecDeque; @@ -1477,7 +1476,7 @@ pub enum UpdateDatabaseResult { /// `None` if the database is in-memory only. durable_offset: Option, }, - AutoMigrateError(Box>), + MigrationPlanningError(Box), ErrorExecutingMigration(anyhow::Error), } impl UpdateDatabaseResult { diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 52abf373e48..c9a9370c863 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -102,9 +102,9 @@ use spacetimedb_client_api_messages::energy::FunctionBudget; use spacetimedb_datastore::locking_tx_datastore::FuncCallType; use spacetimedb_datastore::traits::Program; use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; -use spacetimedb_schema::auto_migrate::MigrationPolicy; use spacetimedb_schema::def::ModuleDef; use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::migrate::MigrationPolicy; use spacetimedb_table::static_assert_size; use std::cell::Cell; use std::num::NonZeroUsize; diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 97951829d16..7f1a4334fc7 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -49,10 +49,10 @@ use spacetimedb_lib::{bsatn, http as st_http, ConnectionId, Hash, ProductType, R use spacetimedb_primitives::{HttpHandlerId, ProcedureId, TableId, ViewFnPtr, ViewId}; use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, Deserialize, ProductValue, Typespace, WithTypespace}; -use spacetimedb_schema::auto_migrate::{MigratePlan, MigrationPolicy, MigrationPolicyError}; use spacetimedb_schema::def::deserialize::FunctionDef; use spacetimedb_schema::def::{ModuleDef, ViewDef}; use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::migrate::{MigratePlan, MigrationPolicy, MigrationPolicyError}; use spacetimedb_schema::reducer_name::ReducerName; use spacetimedb_subscription::SubscriptionPlan; use std::sync::Arc; @@ -661,7 +661,9 @@ impl InstanceCommon { Ok(plan) => plan, Err(e) => { return match e { - MigrationPolicyError::AutoMigrateFailure(e) => Ok(UpdateDatabaseResult::AutoMigrateError(e.into())), + MigrationPolicyError::PlanningFailure(e) => { + Ok(UpdateDatabaseResult::MigrationPlanningError(e.into())) + } _ => Ok(UpdateDatabaseResult::ErrorExecutingMigration(e.into())), } } diff --git a/crates/engine/src/update.rs b/crates/engine/src/update.rs index 98094a69947..16f5813977b 100644 --- a/crates/engine/src/update.rs +++ b/crates/engine/src/update.rs @@ -9,8 +9,8 @@ use spacetimedb_lib::db::auth::StTableType; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::AlgebraicValue; use spacetimedb_primitives::{ColSet, TableId}; -use spacetimedb_schema::auto_migrate::{AutoMigratePlan, AutoMigrateStep, ManualMigratePlan, MigratePlan}; use spacetimedb_schema::def::{ModuleDef, ModuleDefLookup, TableDef, ViewDef}; +use spacetimedb_schema::migrate::{AutoMigratePlan, AutoMigrateStep, ManualMigratePlan, MigratePlan}; use spacetimedb_schema::schema::{ column_schemas_from_defs, ConstraintSchema, IndexSchema, Schema, SequenceSchema, TableSchema, }; @@ -185,7 +185,7 @@ fn auto_migrate_database( for precheck in plan.prechecks { match precheck { - spacetimedb_schema::auto_migrate::AutoMigratePrecheck::CheckAddSequenceRangeValid(sequence_name) => { + spacetimedb_schema::migrate::AutoMigratePrecheck::CheckAddSequenceRangeValid(sequence_name) => { let table_def = plan.new.stored_in_table_def(sequence_name).unwrap(); let sequence_def = &table_def.sequences[sequence_name]; let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); @@ -225,7 +225,7 @@ fn auto_migrate_database( for step in plan.steps { match step { - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveTable(table_name) => { + AutoMigrateStep::RemoveTable(table_name) => { let table_id = stdb.table_id_from_name_mut(tx, table_name)?.unwrap(); if stdb.table_row_count_mut(tx, table_id).unwrap_or(0) > 0 { @@ -238,7 +238,7 @@ fn auto_migrate_database( log!(logger, "Dropping table `{table_name}`"); stdb.drop_table(tx, table_id)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddTable(table_name) => { + AutoMigrateStep::AddTable(table_name) => { let table_def: &TableDef = plan.new.expect_lookup(table_name); // Recursively sets IDs to 0. @@ -249,22 +249,22 @@ fn auto_migrate_database( stdb.create_table(tx, table_schema)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddView(view_name) => { + AutoMigrateStep::AddView(view_name) => { let view_def: &ViewDef = plan.new.expect_lookup(view_name); stdb.create_view(tx, plan.new, view_def)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveView(view_name) => { + AutoMigrateStep::RemoveView(view_name) => { let view_id = stdb.view_id_from_name_mut(tx, view_name)?.unwrap(); stdb.drop_view(tx, view_id)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::UpdateView(_) => { + AutoMigrateStep::UpdateView(_) => { // if we already have to disconnect clients, no need to set // `EvaluateSubscribedViews` as clients will be disconnected anyway if !matches!(res, UpdateResult::RequiresClientDisconnect) { res = UpdateResult::EvaluateSubscribedViews; } } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddIndex(index_name) => { + AutoMigrateStep::AddIndex(index_name) => { let table_def = plan.new.stored_in_table_def(index_name).unwrap(); let index_def = table_def.indexes.get(index_name).unwrap(); let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); @@ -283,7 +283,7 @@ fn auto_migrate_database( stdb.create_index(tx, index_schema, is_unique)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveIndex(index_name) => { + AutoMigrateStep::RemoveIndex(index_name) => { let table_def = plan.old.stored_in_table_def(index_name).unwrap(); let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); @@ -298,7 +298,7 @@ fn auto_migrate_database( log!(logger, "Dropping index `{}` on table `{}`", index_name, table_def.name); stdb.drop_index(tx, index_schema.index_id)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveConstraint(constraint_name) => { + AutoMigrateStep::RemoveConstraint(constraint_name) => { let table_def = plan.old.stored_in_table_def(constraint_name).unwrap(); let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); @@ -317,7 +317,7 @@ fn auto_migrate_database( ); stdb.drop_constraint(tx, constraint_schema.constraint_id)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddConstraint(constraint_name) => { + AutoMigrateStep::AddConstraint(constraint_name) => { let table_def = plan .new .stored_in_table_def(constraint_name) @@ -340,7 +340,7 @@ fn auto_migrate_database( ); stdb.create_constraint(tx, constraint_schema)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddSequence(sequence_name) => { + AutoMigrateStep::AddSequence(sequence_name) => { let table_def = plan.new.stored_in_table_def(sequence_name).unwrap(); let sequence_def = table_def.sequences.get(sequence_name).unwrap(); @@ -357,7 +357,7 @@ fn auto_migrate_database( SequenceSchema::from_module_def(plan.new, sequence_def, table_schema.table_id, 0.into()); stdb.create_sequence(tx, sequence_schema)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveSequence(sequence_name) => { + AutoMigrateStep::RemoveSequence(sequence_name) => { let table_def = plan.old.stored_in_table_def(sequence_name).unwrap(); let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); @@ -376,7 +376,7 @@ fn auto_migrate_database( ); stdb.drop_sequence(tx, sequence_schema.sequence_id)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeColumns(table_name) => { + AutoMigrateStep::ChangeColumns(table_name) => { let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap(); let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id); @@ -385,7 +385,7 @@ fn auto_migrate_database( stdb.alter_table_row_type(tx, table_id, column_schemas)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::ReschemaEventTable(table_name) => { + AutoMigrateStep::ReschemaEventTable(table_name) => { let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap(); let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id); @@ -394,33 +394,33 @@ fn auto_migrate_database( stdb.alter_event_table_row_type(tx, table_id, column_schemas)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeAccess(table_name) => { + AutoMigrateStep::ChangeAccess(table_name) => { let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); stdb.alter_table_access(tx, table_name, table_def.table_access.into())?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangePrimaryKey(table_name) => { + AutoMigrateStep::ChangePrimaryKey(table_name) => { let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); log!(logger, "Changing primary key for table `{table_name}`"); stdb.alter_table_primary_key(tx, table_name, table_def.primary_key)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddSchedule(_) => { + AutoMigrateStep::AddSchedule(_) => { anyhow::bail!("Adding schedules is not yet implemented"); } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveSchedule(_) => { + AutoMigrateStep::RemoveSchedule(_) => { anyhow::bail!("Removing schedules is not yet implemented"); } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddRowLevelSecurity(sql_rls) => { + AutoMigrateStep::AddRowLevelSecurity(sql_rls) => { log!(logger, "Adding row-level security `{sql_rls}`"); let rls = plan.new.lookup_expect(sql_rls); let rls = RowLevelExpr::build_row_level_expr(tx, &auth_ctx, rls)?; stdb.create_row_level_security(tx, rls.def)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveRowLevelSecurity(sql_rls) => { + AutoMigrateStep::RemoveRowLevelSecurity(sql_rls) => { log!(logger, "Removing-row level security `{sql_rls}`"); stdb.drop_row_level_security(tx, sql_rls.clone())?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddColumns(table_name) => { + AutoMigrateStep::AddColumns(table_name) => { let table_def = plan .new .stored_in_table_def(&table_name.clone().into()) @@ -435,7 +435,7 @@ fn auto_migrate_database( .collect(); stdb.add_columns_to_table_mut_tx(tx, table_id, column_schemas, default_values)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::DisconnectAllUsers => { + AutoMigrateStep::DisconnectAllUsers => { log!(logger, "Disconnecting all users"); // It does not disconnect clients right away, // but send response indicated that caller should drop clients diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 1ab04d7a6bd..9982031c77e 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -8,6 +8,7 @@ use crate::db::raw_def::v9::{Lifecycle, RawIndexAlgorithm, TableAccess, TableType}; use core::fmt; use spacetimedb_primitives::{ColId, ColList}; +use spacetimedb_sats::hash::Hash; use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::typespace::TypespaceBuilder; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, AlgebraicValue, ProductType, SpacetimeType, Typespace}; @@ -98,6 +99,9 @@ pub enum RawModuleDefV10Section { /// Primary key metadata for views. ViewPrimaryKeys(Vec), + + /// Guest-defined logic for manual migrations. + ManualMigrationFunctions(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -557,6 +561,19 @@ pub struct RawViewPrimaryKeyDefV10 { pub columns: Vec, } +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawManualMigrationFunctionDefV10 { + pub function_source_name: RawIdentifier, + /// The Blake3 hash of the raw module def of the module version from which this function migrates. + /// + /// Stored here as a [`Hash`] rather than a [`blake3::Hash`] as the former type implements [`SpacetimeType`] and [`Ord`]. + // TODO(manual-migrations): determine precise semantics for computing this. + // Is it always the hash of the BSATN-ified `RawModuleDefV10`, even if the previous module exports a `RawModuleDefV9` or earlier? + pub previous_raw_module_def_bsatn_blake3_hash: Hash, +} + impl RawModuleDefV10 { /// Get the types section, if present. pub fn types(&self) -> Option<&Vec> { @@ -678,6 +695,13 @@ impl RawModuleDefV10 { _ => None, }) } + + pub fn manual_migration_functions(&self) -> Option<&Vec> { + self.sections.iter().find_map(|s| match s { + RawModuleDefV10Section::ManualMigrationFunctions(migrations) => Some(migrations), + _ => None, + }) + } } /// A builder for a [`RawModuleDefV10`]. @@ -935,6 +959,26 @@ impl RawModuleDefV10Builder { } } + /// Get mutable access to the manual migration functions section, creating it if missing. + fn manual_migration_functions_mut(&mut self) -> &mut Vec { + let idx = self + .module + .sections + .iter() + .position(|s| matches!(s, RawModuleDefV10Section::ManualMigrationFunctions(_))) + .unwrap_or_else(|| { + self.module + .sections + .push(RawModuleDefV10Section::ManualMigrationFunctions(Vec::new())); + self.module.sections.len() - 1 + }); + + match &mut self.module.sections[idx] { + RawModuleDefV10Section::ManualMigrationFunctions(migrations) => migrations, + _ => unreachable!("Just ensured ManualMigrationFunctions section exists"), + } + } + /// Create a table builder. /// /// Does not validate that the product_type_ref is valid; this is left to the module validation code. @@ -1217,6 +1261,18 @@ impl RawModuleDefV10Builder { self.explicit_names_mut().merge(names); } + pub fn add_manual_migration_function( + &mut self, + source_name: impl Into, + previous_raw_module_def_bsatn_blake3_hash: Hash, + ) { + self.manual_migration_functions_mut() + .push(RawManualMigrationFunctionDefV10 { + function_source_name: source_name.into(), + previous_raw_module_def_bsatn_blake3_hash, + }); + } + /// Set the case conversion policy for this module. /// /// By default, SpacetimeDB applies `SnakeCase` conversion to table names, diff --git a/crates/schema/Cargo.toml b/crates/schema/Cargo.toml index 25ff3e3e9cc..d72d4f7d4dc 100644 --- a/crates/schema/Cargo.toml +++ b/crates/schema/Cargo.toml @@ -17,6 +17,7 @@ spacetimedb-data-structures.workspace = true spacetimedb-memory-usage.workspace = true spacetimedb-sql-parser.workspace = true anyhow.workspace = true +blake3.workspace = true derive_more.workspace = true indexmap.workspace = true itertools.workspace = true diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 9ad07dce72f..c1608917884 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -33,9 +33,9 @@ use spacetimedb_data_structures::map::{Equivalent, HashMap}; use spacetimedb_lib::db::raw_def; use spacetimedb_lib::db::raw_def::v10::{ ExplicitNames, MethodOrAny, RawConstraintDefV10, RawHttpHandlerDefV10, RawHttpRouteDefV10, RawIndexDefV10, - RawLifeCycleReducerDefV10, RawModuleDefV10, RawModuleDefV10Section, RawProcedureDefV10, RawReducerDefV10, - RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, RawSequenceDefV10, RawTableDefV10, - RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, + RawLifeCycleReducerDefV10, RawManualMigrationFunctionDefV10, RawModuleDefV10, RawModuleDefV10Section, + RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, + RawSequenceDefV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, }; use spacetimedb_lib::db::raw_def::v9::{ Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9, @@ -44,7 +44,7 @@ use spacetimedb_lib::db::raw_def::v9::{ RawUniqueConstraintDataV9, RawViewDefV9, TableAccess, TableType, }; use spacetimedb_lib::db::view::{extract_view_return_product_type_ref, ViewKind}; -use spacetimedb_lib::{ProductType, RawModuleDef}; +use spacetimedb_lib::{bsatn, ProductType, RawModuleDef}; use spacetimedb_primitives::{ ColId, ColList, ColOrCols, ColSet, HttpHandlerId, ProcedureId, ReducerId, TableId, ViewFnPtr, }; @@ -164,6 +164,13 @@ pub struct ModuleDef { /// was authored under. #[allow(unused)] raw_module_def_version: RawModuleDefVersion, + + /// Manual migration functions defined by the module, + /// keyed on the Blake3 hash of the raw module def of the previous module version. + /// + /// Uses [`IndexMap`] to preserve order so that `__call_manual_migration_function__` + /// receives stable integer IDs. + manual_migration_functions: IndexMap, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -460,6 +467,24 @@ impl ModuleDef { panic!("expected ModuleDef to contain {:?}, but it does not", def.key()); } } + + pub fn manual_migration_function_for_previous_module_def_hash( + &self, + prev_hash: spacetimedb_lib::Hash, + ) -> Option<&ManualMigrationFunctionDef> { + self.manual_migration_functions.get(&prev_hash) + } + + /// Compute the hash of this module def, + /// for use as an identifier a manual migration can use to reference its expected previous state. + /// + /// This takes the Blake3 hash of the BSATN bytes of the `RawModuleDefV10` representation of `self`. + pub fn manual_migration_hash(&self) -> Result { + let raw_def = RawModuleDefV10::from(self.clone()); + let bsatn_bytes = bsatn::to_vec(&raw_def)?; + let hash = blake3::hash(&bsatn_bytes); + Ok(spacetimedb_lib::Hash::from_byte_array(*hash.as_bytes())) + } } impl TryFrom for ModuleDef { @@ -507,6 +532,7 @@ impl From for RawModuleDefV9 { http_handlers: _, http_routes: _, raw_module_def_version: _, + manual_migration_functions: _, } = val; // Extract column defaults from tables before consuming tables @@ -565,6 +591,7 @@ impl From for RawModuleDefV10 { http_handlers, http_routes, raw_module_def_version: _, + manual_migration_functions: _, } = val; let mut sections = Vec::new(); @@ -1911,6 +1938,25 @@ impl From for RawMiscModuleExportV9 { } } +#[derive(Debug, Clone, Eq, PartialEq)] +#[non_exhaustive] +pub struct ManualMigrationFunctionDef { + pub function_source_name: RawIdentifier, + /// The Blake3 hash of the raw module def of the module version from which this function migrates. + // TODO(manual-migrations): determine precise semantics for computing this. + // Is it always the hash of the BSATN-ified `RawModuleDefV10`, even if the previous module exports a `RawModuleDefV9` or earlier? + pub previous_raw_module_def_bsatn_blake3_hash: spacetimedb_lib::Hash, +} + +impl From for RawManualMigrationFunctionDefV10 { + fn from(def: ManualMigrationFunctionDef) -> Self { + RawManualMigrationFunctionDefV10 { + function_source_name: def.function_source_name, + previous_raw_module_def_bsatn_blake3_hash: def.previous_raw_module_def_bsatn_blake3_hash, + } + } +} + impl ModuleDefLookup for TableDef { type Key<'a> = &'a Identifier; @@ -2090,6 +2136,16 @@ impl ModuleDefLookup for ViewDef { } } +impl ModuleDefLookup for ManualMigrationFunctionDef { + type Key<'a> = &'a spacetimedb_lib::Hash; + fn key(&self) -> Self::Key<'_> { + &self.previous_raw_module_def_bsatn_blake3_hash + } + fn lookup<'def>(module_def: &'def ModuleDef, key: Self::Key<'_>) -> Option<&'def Self> { + module_def.manual_migration_functions.get(key) + } +} + fn to_raw(data: HashMap) -> Vec where Def: ModuleDefLookup + Into, diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index aa398743f97..1a03149ae76 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -248,6 +248,10 @@ pub fn validate(def: RawModuleDefV10) -> Result { validate_http_routes_are_unique(&routes)?; Ok((handlers, routes)) }); + + let migration_functions = + validate_migration_function_hashes_are_unique(def.manual_migration_functions().unwrap_or(&Vec::new())); + // Combine all validation results let tables_types_reducers_procedures_views = ( tables, @@ -258,10 +262,21 @@ pub fn validate(def: RawModuleDefV10) -> Result { schedules, lifecycle_validations, http_handlers_and_routes, + migration_functions, ) .combine_errors() .and_then( - |(mut tables, types, reducers, procedures, views, schedules, lifecycles, http_handlers_and_routes)| { + |( + mut tables, + types, + reducers, + procedures, + views, + schedules, + lifecycles, + http_handlers_and_routes, + migration_functions, + )| { let (mut reducers, mut procedures, mut views) = check_function_names_are_unique(reducers, procedures, views)?; // Attach lifecycles to their respective reducers @@ -275,7 +290,15 @@ pub fn validate(def: RawModuleDefV10) -> Result { attach_view_primary_keys(&mut views, view_primary_keys)?; assign_query_view_primary_keys(&tables, &mut views); - Ok((tables, types, reducers, procedures, views, http_handlers_and_routes)) + Ok(( + tables, + types, + reducers, + procedures, + views, + http_handlers_and_routes, + migration_functions, + )) }, ); let CoreValidator { @@ -292,11 +315,20 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(|rls| (rls.sql.clone(), rls.to_owned())) .collect(); - let (tables, types, reducers, procedures, views, http_handlers, http_routes) = + let (tables, types, reducers, procedures, views, http_handlers, http_routes, migration_functions) = tables_types_reducers_procedures_views .map( - |(tables, types, reducers, procedures, views, (http_handlers, http_routes))| { - (tables, types, reducers, procedures, views, http_handlers, http_routes) + |(tables, types, reducers, procedures, views, (http_handlers, http_routes), migration_functions)| { + ( + tables, + types, + reducers, + procedures, + views, + http_handlers, + http_routes, + migration_functions, + ) }, ) .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; @@ -318,6 +350,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { http_handlers, http_routes, raw_module_def_version: RawModuleDefVersion::V10, + manual_migration_functions: migration_functions, }) } @@ -412,6 +445,34 @@ fn check_http_handler_names_are_unique( ErrorStream::add_extra_errors(Ok(handlers_map), errors) } +fn validate_migration_function_hashes_are_unique( + migrations: &[RawManualMigrationFunctionDefV10], +) -> Result> { + let mut errors = vec![]; + let mut migrations_map: IndexMap = + IndexMap::with_capacity(migrations.len()); + + for migration in migrations { + if let Some(other_migration) = migrations_map.get(&migration.previous_raw_module_def_bsatn_blake3_hash) { + errors.push(ValidationError::DuplicateManualMigrationPreviousHash { + previous_raw_module_def_bsatn_blake3_hash: migration.previous_raw_module_def_bsatn_blake3_hash, + function_source_name_a: other_migration.function_source_name.clone(), + function_source_name_b: migration.function_source_name.clone(), + }); + } else { + migrations_map.insert( + migration.previous_raw_module_def_bsatn_blake3_hash, + ManualMigrationFunctionDef { + function_source_name: migration.function_source_name.clone(), + previous_raw_module_def_bsatn_blake3_hash: migration.previous_raw_module_def_bsatn_blake3_hash, + }, + ); + } + } + + ErrorStream::add_extra_errors(Ok(migrations_map), errors) +} + struct ModuleValidatorV10<'a> { core: CoreValidator<'a>, } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 618f8e3c9c4..388197ab90d 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -168,6 +168,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { http_handlers: IndexMap::new(), http_routes: Vec::new(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, + manual_migration_functions: IndexMap::new(), }) } diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 0bab5445d92..95ac6c26abf 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -176,6 +176,12 @@ pub enum ValidationError { ok_type: PrettyAlgebraicType, err_type: PrettyAlgebraicType, }, + #[error("multiple manual migration functions defined to update from module def hash {previous_raw_module_def_bsatn_blake3_hash}, with identifiers {function_source_name_a} and {function_source_name_b}")] + DuplicateManualMigrationPreviousHash { + previous_raw_module_def_bsatn_blake3_hash: spacetimedb_lib::Hash, + function_source_name_a: RawIdentifier, + function_source_name_b: RawIdentifier, + }, } /// A wrapper around an `AlgebraicType` that implements `fmt::Display`. diff --git a/crates/schema/src/lib.rs b/crates/schema/src/lib.rs index aa703e37b2d..56996dbf24d 100644 --- a/crates/schema/src/lib.rs +++ b/crates/schema/src/lib.rs @@ -2,10 +2,10 @@ //! //! Handles validation and normalization of raw schema definitions from the `spacetimedb_lib` crate. -pub mod auto_migrate; pub mod def; pub mod error; pub mod identifier; +pub mod migrate; pub mod reducer_name; pub mod relation; pub mod schema; diff --git a/crates/schema/src/migrate.rs b/crates/schema/src/migrate.rs new file mode 100644 index 00000000000..befbeeb0013 --- /dev/null +++ b/crates/schema/src/migrate.rs @@ -0,0 +1,217 @@ +use auto_migrate::{format_plan, ponder_auto_migrate, ColorScheme, TermColorFormatter}; +use manual_migrate::{ponder_manual_migrate, ManualMigrateError}; +use spacetimedb_data_structures::error_stream::ErrorStream; +use spacetimedb_lib::{bsatn, hash_bytes, Identity}; +use thiserror::Error; + +use crate::def::ModuleDef; + +mod auto_migrate; +mod manual_migrate; + +pub use auto_migrate::{AutoMigrateError, AutoMigratePlan, AutoMigratePrecheck, AutoMigrateStep}; +pub use manual_migrate::ManualMigratePlan; + +/// A plan for a migration. +#[derive(Debug)] +pub enum MigratePlan<'def> { + Manual(ManualMigratePlan<'def>), + Auto(AutoMigratePlan<'def>), +} + +#[derive(Copy, Clone, PartialEq, Eq)] +pub enum PrettyPrintStyle { + AnsiColor, + NoColor, +} + +impl<'def> MigratePlan<'def> { + /// Get the old `ModuleDef` for this migration plan. + pub fn old_def(&self) -> &'def ModuleDef { + match self { + MigratePlan::Manual(plan) => plan.old, + MigratePlan::Auto(plan) => plan.old, + } + } + + /// Get the new `ModuleDef` for this migration plan. + pub fn new_def(&self) -> &'def ModuleDef { + match self { + MigratePlan::Manual(plan) => plan.new, + MigratePlan::Auto(plan) => plan.new, + } + } + + pub fn breaks_client(&self) -> bool { + match self { + //TODO: fix it when support for manual migration plans is added. + MigratePlan::Manual(_) => true, + MigratePlan::Auto(plan) => plan + .steps + .iter() + .any(|step| matches!(step, AutoMigrateStep::DisconnectAllUsers)), + } + } + + pub fn pretty_print(&self, style: PrettyPrintStyle) -> anyhow::Result { + use PrettyPrintStyle::*; + match self { + MigratePlan::Manual(_) => { + anyhow::bail!("Manual migration plans are not yet supported for pretty printing.") + } + + MigratePlan::Auto(plan) => match style { + NoColor => { + let mut fmt = TermColorFormatter::new(ColorScheme::default(), termcolor::ColorChoice::Never); + format_plan(&mut fmt, plan).map(|_| fmt.to_string()) + } + AnsiColor => { + let mut fmt = TermColorFormatter::new(ColorScheme::default(), termcolor::ColorChoice::AlwaysAnsi); + format_plan(&mut fmt, plan).map(|_| fmt.to_string()) + } + } + .map_err(|e| anyhow::anyhow!("Failed to format migration plan: {e}")), + } + } +} + +/// A migration policy that determines whether a module update is allowed to break client compatibility. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MigrationPolicy { + /// Migration must maintain backward compatibility with existing clients. + Compatible, + /// To use this, a valid [`MigrationToken`] must be provided. + /// The token is issued through the pre-publish API (see the `client-api` crate) + /// and proves that the publisher explicitly acknowledged the breaking change. + BreakClients(spacetimedb_lib::Hash), +} + +impl MigrationPolicy { + /// Verifies whether the given migration plan is allowed under the current policy. + /// + /// Returns `Ok(())` if allowed, otherwise an appropriate `MigrationPolicyError` + fn permits_plan(&self, plan: &MigratePlan<'_>, token: &MigrationToken) -> anyhow::Result<(), MigrationPolicyError> { + match self { + MigrationPolicy::Compatible => { + if plan.breaks_client() { + Err(MigrationPolicyError::ClientBreakingChangeDisallowed) + } else { + Ok(()) + } + } + MigrationPolicy::BreakClients(expected_hash) => { + if token.hash() == *expected_hash { + Ok(()) + } else { + Err(MigrationPolicyError::InvalidToken) + } + } + } + } + + /// Attempts to generate a migration plan and validate it under this policy. + /// + /// Fails if migration is not permitted by the policy or migration planning fails. + pub fn try_migrate<'def>( + &self, + database_identity: Identity, + old_module_hash: spacetimedb_lib::Hash, + old_module_def: &'def ModuleDef, + new_module_hash: spacetimedb_lib::Hash, + new_module_def: &'def ModuleDef, + ) -> anyhow::Result, MigrationPolicyError> { + let plan = ponder_migrate(old_module_def, new_module_def).map_err(MigrationPolicyError::PlanningFailure)?; + self.permits_migrate_plan(database_identity, old_module_hash, new_module_hash, &plan)?; + Ok(plan) + } + + /// Validate an already-generated migration plan under this policy. + pub fn permits_migrate_plan( + &self, + database_identity: Identity, + old_module_hash: spacetimedb_lib::Hash, + new_module_hash: spacetimedb_lib::Hash, + plan: &MigratePlan<'_>, + ) -> anyhow::Result<(), MigrationPolicyError> { + let token = MigrationToken { + database_identity, + old_module_hash, + new_module_hash, + }; + self.permits_plan(plan, &token)?; + Ok(()) + } +} + +#[derive(Debug, Error)] +pub enum MigrationPolicyError { + #[error("Migration planning failed: {0}")] + PlanningFailure(PonderMigrateError), + + #[error("Token provided is invalid or does not match expected hash")] + InvalidToken, + + #[error("Migration plan contains a client-breaking change which is disallowed under current policy")] + ClientBreakingChangeDisallowed, +} + +/// A token acknowledging a breaking migration. +/// +/// Note: This token is only intended as a UX safeguard, not as a security measure. +/// No secret is used in its generation, which means anyone can reproduce it given +/// the inputs. That is acceptable for our purposes since it only signals user intent, +/// not authorization. +pub struct MigrationToken { + pub database_identity: Identity, + pub old_module_hash: spacetimedb_lib::Hash, + pub new_module_hash: spacetimedb_lib::Hash, +} + +impl MigrationToken { + pub fn hash(&self) -> spacetimedb_lib::Hash { + hash_bytes( + format!( + "{}{}{}", + self.database_identity.to_hex(), + self.old_module_hash.to_hex(), + self.new_module_hash.to_hex() + ) + .as_str(), + ) + } +} + +#[derive(thiserror::Error, Debug)] +pub enum PonderMigrateError { + #[error("ModuleDef changes require a manual migration, but new `ModuleDef` does not specify a manual migration function from previous `ModuleDef` {old_hash}.\nManual migration is required because {errors}")] + AutoMigrate { + errors: ErrorStream, + old_hash: spacetimedb_lib::Hash, + }, + #[error("Planning manual migration failed: ")] + ManualMigrate(#[from] ErrorStream), + #[error("Error while BSATN encoding old `ModuleDef` in order to compute its hash when pondering a manual migration: {0}")] + Bsatn(#[from] bsatn::EncodeError), +} + +/// Construct a migration plan. +/// If `new` has an `__update__` reducer, return a manual migration plan. +/// Otherwise, try to plan an automatic migration. This may fail. +pub fn ponder_migrate<'def>( + old: &'def ModuleDef, + new: &'def ModuleDef, +) -> Result, PonderMigrateError> { + let old_hash = old.manual_migration_hash()?; + if new + .manual_migration_function_for_previous_module_def_hash(old_hash) + .is_some() + { + ponder_manual_migrate(old, old_hash, new) + .map(MigratePlan::Manual) + .map_err(PonderMigrateError::ManualMigrate) + } else { + ponder_auto_migrate(old, new) + .map(MigratePlan::Auto) + .map_err(|errors| PonderMigrateError::AutoMigrate { errors, old_hash }) + } +} diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/migrate/auto_migrate.rs similarity index 93% rename from crates/schema/src/auto_migrate.rs rename to crates/schema/src/migrate/auto_migrate.rs index 63239de7ecc..d712ca7f522 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/migrate/auto_migrate.rs @@ -1,204 +1,23 @@ -use core::{cmp::Ordering, ops::BitOr}; - use crate::{def::*, error::PrettyAlgebraicType, identifier::Identifier}; -use formatter::format_plan; +use core::{cmp::Ordering, ops::BitOr}; +pub use formatter::format_plan; use spacetimedb_data_structures::{ error_stream::{CollectAllErrors, CombineErrors, ErrorStream}, map::{HashCollectionExt as _, HashSet}, }; -use spacetimedb_lib::{ - db::raw_def::v9::{RawRowLevelSecurityDefV9, TableType}, - hash_bytes, Identity, -}; +use spacetimedb_lib::db::raw_def::v9::{RawRowLevelSecurityDefV9, TableType}; use spacetimedb_sats::{ layout::{HasLayout, SumTypeLayout}, raw_identifier::RawIdentifier, AlgebraicType, WithTypespace, }; -use termcolor_formatter::{ColorScheme, TermColorFormatter}; -use thiserror::Error; +pub use termcolor_formatter::{ColorScheme, TermColorFormatter}; + mod formatter; mod termcolor_formatter; pub type Result = std::result::Result>; -/// A plan for a migration. -#[derive(Debug)] -pub enum MigratePlan<'def> { - Manual(ManualMigratePlan<'def>), - Auto(AutoMigratePlan<'def>), -} - -#[derive(Copy, Clone, PartialEq, Eq)] -pub enum PrettyPrintStyle { - AnsiColor, - NoColor, -} - -impl<'def> MigratePlan<'def> { - /// Get the old `ModuleDef` for this migration plan. - pub fn old_def(&self) -> &'def ModuleDef { - match self { - MigratePlan::Manual(plan) => plan.old, - MigratePlan::Auto(plan) => plan.old, - } - } - - /// Get the new `ModuleDef` for this migration plan. - pub fn new_def(&self) -> &'def ModuleDef { - match self { - MigratePlan::Manual(plan) => plan.new, - MigratePlan::Auto(plan) => plan.new, - } - } - - pub fn breaks_client(&self) -> bool { - match self { - //TODO: fix it when support for manual migration plans is added. - MigratePlan::Manual(_) => true, - MigratePlan::Auto(plan) => plan - .steps - .iter() - .any(|step| matches!(step, AutoMigrateStep::DisconnectAllUsers)), - } - } - - pub fn pretty_print(&self, style: PrettyPrintStyle) -> anyhow::Result { - use PrettyPrintStyle::*; - match self { - MigratePlan::Manual(_) => { - anyhow::bail!("Manual migration plans are not yet supported for pretty printing.") - } - - MigratePlan::Auto(plan) => match style { - NoColor => { - let mut fmt = TermColorFormatter::new(ColorScheme::default(), termcolor::ColorChoice::Never); - format_plan(&mut fmt, plan).map(|_| fmt.to_string()) - } - AnsiColor => { - let mut fmt = TermColorFormatter::new(ColorScheme::default(), termcolor::ColorChoice::AlwaysAnsi); - format_plan(&mut fmt, plan).map(|_| fmt.to_string()) - } - } - .map_err(|e| anyhow::anyhow!("Failed to format migration plan: {e}")), - } - } -} - -/// A migration policy that determines whether a module update is allowed to break client compatibility. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum MigrationPolicy { - /// Migration must maintain backward compatibility with existing clients. - Compatible, - /// To use this, a valid [`MigrationToken`] must be provided. - /// The token is issued through the pre-publish API (see the `client-api` crate) - /// and proves that the publisher explicitly acknowledged the breaking change. - BreakClients(spacetimedb_lib::Hash), -} - -impl MigrationPolicy { - /// Verifies whether the given migration plan is allowed under the current policy. - /// - /// Returns `Ok(())` if allowed, otherwise an appropriate `MigrationPolicyError` - fn permits_plan(&self, plan: &MigratePlan<'_>, token: &MigrationToken) -> anyhow::Result<(), MigrationPolicyError> { - match self { - MigrationPolicy::Compatible => { - if plan.breaks_client() { - Err(MigrationPolicyError::ClientBreakingChangeDisallowed) - } else { - Ok(()) - } - } - MigrationPolicy::BreakClients(expected_hash) => { - if token.hash() == *expected_hash { - Ok(()) - } else { - Err(MigrationPolicyError::InvalidToken) - } - } - } - } - - /// Attempts to generate a migration plan and validate it under this policy. - /// - /// Fails if migration is not permitted by the policy or migration planning fails. - pub fn try_migrate<'def>( - &self, - database_identity: Identity, - old_module_hash: spacetimedb_lib::Hash, - old_module_def: &'def ModuleDef, - new_module_hash: spacetimedb_lib::Hash, - new_module_def: &'def ModuleDef, - ) -> anyhow::Result, MigrationPolicyError> { - let plan = ponder_migrate(old_module_def, new_module_def).map_err(MigrationPolicyError::AutoMigrateFailure)?; - self.permits_migrate_plan(database_identity, old_module_hash, new_module_hash, &plan)?; - Ok(plan) - } - - /// Validate an already-generated migration plan under this policy. - pub fn permits_migrate_plan( - &self, - database_identity: Identity, - old_module_hash: spacetimedb_lib::Hash, - new_module_hash: spacetimedb_lib::Hash, - plan: &MigratePlan<'_>, - ) -> anyhow::Result<(), MigrationPolicyError> { - let token = MigrationToken { - database_identity, - old_module_hash, - new_module_hash, - }; - self.permits_plan(plan, &token)?; - Ok(()) - } -} - -#[derive(Debug, Error)] -pub enum MigrationPolicyError { - #[error("Automatic migration planning failed")] - AutoMigrateFailure(ErrorStream), - - #[error("Token provided is invalid or does not match expected hash")] - InvalidToken, - - #[error("Migration plan contains a client-breaking change which is disallowed under current policy")] - ClientBreakingChangeDisallowed, -} - -/// A token acknowledging a breaking migration. -/// -/// Note: This token is only intended as a UX safeguard, not as a security measure. -/// No secret is used in its generation, which means anyone can reproduce it given -/// the inputs. That is acceptable for our purposes since it only signals user intent, -/// not authorization. -pub struct MigrationToken { - pub database_identity: Identity, - pub old_module_hash: spacetimedb_lib::Hash, - pub new_module_hash: spacetimedb_lib::Hash, -} - -impl MigrationToken { - pub fn hash(&self) -> spacetimedb_lib::Hash { - hash_bytes( - format!( - "{}{}{}", - self.database_identity.to_hex(), - self.old_module_hash.to_hex(), - self.new_module_hash.to_hex() - ) - .as_str(), - ) - } -} - -/// A plan for a manual migration. -/// `new` must have a reducer marked with `Lifecycle::Update`. -#[derive(Debug)] -pub struct ManualMigratePlan<'def> { - pub old: &'def ModuleDef, - pub new: &'def ModuleDef, -} - /// A plan for an automatic migration. #[derive(Debug)] pub struct AutoMigratePlan<'def> { @@ -276,6 +95,8 @@ pub enum AutoMigrateStep<'def> { RemoveRowLevelSecurity(::Key<'def>), /// Remove an empty table and all its sub-objects (indexes, constraints, sequences). /// Validated at execution time: fails if the table contains data. + /// + /// Will never appear in a [`super::manual_migrate::ManualMigratePlan::auto_migrate_steps_before`]. RemoveTable(::Key<'def>), /// Change the column types of a table, in a layout compatible way. @@ -463,15 +284,6 @@ pub enum AutoMigrateError { }, } -/// Construct a migration plan. -/// If `new` has an `__update__` reducer, return a manual migration plan. -/// Otherwise, try to plan an automatic migration. This may fail. -pub fn ponder_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> Result> { - // TODO(1.0): Implement this function. - // Currently we only can do automatic migrations. - ponder_auto_migrate(old, new).map(MigratePlan::Auto) -} - /// Construct an automatic migration plan, or reject with reasons why automatic migration can't be performed. pub fn ponder_auto_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> Result> { // Both the old and new database definitions have already been validated (this is enforced by the types). diff --git a/crates/schema/src/auto_migrate/formatter.rs b/crates/schema/src/migrate/auto_migrate/formatter.rs similarity index 99% rename from crates/schema/src/auto_migrate/formatter.rs rename to crates/schema/src/migrate/auto_migrate/formatter.rs index cd6c1042077..bd1f9c94f5d 100644 --- a/crates/schema/src/auto_migrate/formatter.rs +++ b/crates/schema/src/migrate/auto_migrate/formatter.rs @@ -4,9 +4,9 @@ use std::io; use super::{AutoMigratePlan, ModuleDefLookup, TableDef}; use crate::{ - auto_migrate::AutoMigrateStep, def::{ConstraintData, FunctionKind, ModuleDef, ScheduleDef, ViewDef}, identifier::Identifier, + migrate::auto_migrate::AutoMigrateStep, }; use itertools::Itertools; use spacetimedb_lib::db::raw_def::v9::{RawRowLevelSecurityDefV9, TableAccess, TableType}; diff --git a/crates/schema/src/auto_migrate/termcolor_formatter.rs b/crates/schema/src/migrate/auto_migrate/termcolor_formatter.rs similarity index 99% rename from crates/schema/src/auto_migrate/termcolor_formatter.rs rename to crates/schema/src/migrate/auto_migrate/termcolor_formatter.rs index 3f9764cfabf..f5cee223644 100644 --- a/crates/schema/src/auto_migrate/termcolor_formatter.rs +++ b/crates/schema/src/migrate/auto_migrate/termcolor_formatter.rs @@ -6,8 +6,8 @@ use spacetimedb_primitives::ColId; use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; use termcolor::{Buffer, Color, ColorChoice, ColorSpec, WriteColor}; -use crate::auto_migrate::formatter::ViewInfo; use crate::identifier::Identifier; +use crate::migrate::auto_migrate::formatter::ViewInfo; use super::formatter::{ AccessChangeInfo, Action, ColumnChange, ColumnChanges, ConstraintInfo, IndexInfo, MigrationFormatter, NewColumns, diff --git a/crates/schema/src/migrate/manual_migrate.rs b/crates/schema/src/migrate/manual_migrate.rs new file mode 100644 index 00000000000..e7674f806db --- /dev/null +++ b/crates/schema/src/migrate/manual_migrate.rs @@ -0,0 +1,33 @@ +use spacetimedb_data_structures::error_stream::ErrorStream; + +use crate::{ + def::{ManualMigrationFunctionDef, ModuleDef, ModuleDefLookup, TableDef}, + migrate::auto_migrate::AutoMigrateStep, +}; + +type Ident<'def, ModuleDefEntry> = ::Key<'def>; + +/// A plan for a manual migration. +#[derive(Debug)] +pub struct ManualMigratePlan<'def> { + pub old: &'def ModuleDef, + pub new: &'def ModuleDef, + pub tables_to_rename_before_and_delete_after: Vec>, + /// Auto-migrate steps to execute before running the manual migration function. + /// + /// This must never contain an [`AutoMigrateStep::RemoveTable`]. + /// Tables to be removed will instead be listed in [`Self::tables_to_rename_before_and_delete_after`]. + pub auto_migrate_steps_before: Vec>, + pub manual_migration_function: Ident<'def, ManualMigrationFunctionDef>, +} + +#[derive(thiserror::Error, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum ManualMigrateError {} + +pub fn ponder_manual_migrate<'def>( + old: &'def ModuleDef, + old_hash: spacetimedb_lib::Hash, + new: &'def ModuleDef, +) -> Result, ErrorStream> { + todo!() +} diff --git a/crates/standalone/src/lib.rs b/crates/standalone/src/lib.rs index 541dd5b6a39..a3565160bc9 100644 --- a/crates/standalone/src/lib.rs +++ b/crates/standalone/src/lib.rs @@ -32,7 +32,7 @@ use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::traits::Program; use spacetimedb_paths::server::{ModuleLogsDir, PidFile, ServerDataDir}; use spacetimedb_paths::standalone::StandaloneDataDirExt; -use spacetimedb_schema::auto_migrate::{MigrationPolicy, PrettyPrintStyle}; +use spacetimedb_schema::migrate::{MigrationPolicy, PrettyPrintStyle}; use spacetimedb_table::page_pool::PagePool; use std::sync::Arc; use std::time::Duration; diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index b03b87df9b5..e81618ef6d3 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -15,8 +15,8 @@ use spacetimedb::Identity; use spacetimedb_client_api::auth::SpacetimeAuth; use spacetimedb_client_api::routes::subscribe::{generate_random_connection_id, WebSocketOptions}; use spacetimedb_paths::{RootDir, SpacetimePaths}; -use spacetimedb_schema::auto_migrate::MigrationPolicy; use spacetimedb_schema::def::ModuleDef; +use spacetimedb_schema::migrate::MigrationPolicy; use tokio::runtime::{Builder, Runtime}; use spacetimedb::client::messages::SerializableMessage;