Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion crates/nokv-client/src/append_operations_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,6 @@ fn inspection(observation: OperationStatus, start: u32, end: u32) -> AppendClean
object_identity: ObjectIdentity::new(format!("object-{sequence}")).unwrap(),
expected_length: 4,
expected_digest: sha256_digest_uri(Digest([7; 32])),
multipart_token: None,
})
.collect(),
next_after: (end < 5).then(|| end - 1),
Expand Down
1 change: 0 additions & 1 deletion crates/nokv-client/src/artifact.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3181,7 +3181,6 @@ fn publication_rows(
object_identity: object_identity.clone(),
expected_length: block.len,
expected_digest: digest.clone(),
multipart_token: None,
});
manifest_rows.push(ArtifactManifestRow {
object_index: block.object_index,
Expand Down
4 changes: 2 additions & 2 deletions crates/nokv-meta/src/workspace/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,8 @@ pub use publish_operation_records::{
ArtifactManifestRow, ManifestPosition, PublishAuthority, PublishClaim, PublishOperationRecord,
PublishRecordError, PublishResult, PublishTerminalError, PublishTerminalErrorKind,
PublishTransition, StagedObjectRecord, MAX_APPEND_SEGMENTS, MAX_MANIFEST_ROWS,
MAX_MULTIPART_ID_BYTES, MAX_STAGED_OBJECTS, MAX_TERMINAL_ERROR_BYTES,
PUBLISH_OPERATION_VALUE_FORMAT_VERSION, PUBLISH_VALUE_FORMAT_VERSION,
MAX_STAGED_OBJECTS, MAX_TERMINAL_ERROR_BYTES, PUBLISH_OPERATION_VALUE_FORMAT_VERSION,
PUBLISH_VALUE_FORMAT_VERSION, STAGED_OBJECT_VALUE_FORMAT_VERSION,
};
pub use query::{
aggregate_paths_at, catalog_fields_at, find_workspaces_at, get_current_workspace,
Expand Down
14 changes: 1 addition & 13 deletions crates/nokv-meta/src/workspace/publication.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1014,7 +1014,6 @@ fn validate_cleanup_update(
if expected.artifact_revision_id != next.artifact_revision_id
|| expected.object_sequence != next.object_sequence
|| expected.object_key != next.object_key
|| expected.multipart_upload_id != next.multipart_upload_id
|| expected.expected_length != next.expected_length
|| expected.expected_digest_uri != next.expected_digest_uri
|| next.provider_state != StagedProviderState::Aborted
Expand Down Expand Up @@ -1044,10 +1043,6 @@ fn validate_staged_update(
|| expected.object_key != next.object_key
|| expected.expected_length != next.expected_length
|| expected.expected_digest_uri != next.expected_digest_uri
|| expected
.multipart_upload_id
.as_ref()
.is_some_and(|id| next.multipart_upload_id.as_ref() != Some(id))
|| !valid_provider_transition(expected.provider_state, next.provider_state)
|| !valid_cleanup_transition(expected.cleanup_state, next.cleanup_state)
{
Expand All @@ -1064,12 +1059,6 @@ fn valid_provider_transition(expected: StagedProviderState, next: StagedProvider
(expected, next),
(
StagedProviderState::Planned,
StagedProviderState::Uploading
| StagedProviderState::Uploaded
| StagedProviderState::AbortPending
| StagedProviderState::Ambiguous
) | (
StagedProviderState::Uploading,
StagedProviderState::Uploaded
| StagedProviderState::AbortPending
| StagedProviderState::Ambiguous
Expand Down Expand Up @@ -5324,7 +5313,6 @@ mod tests {
artifact_revision_id,
u64::from(sequence),
),
multipart_upload_id: None,
expected_length: 1,
expected_digest_uri: format!("sha256:{sequence:064x}"),
provider_state: StagedProviderState::Planned,
Expand Down Expand Up @@ -8058,7 +8046,7 @@ mod tests {

for next in [
StagedObjectRecord {
provider_state: StagedProviderState::Uploading,
provider_state: StagedProviderState::Planned,
..persisted.clone()
},
StagedObjectRecord {
Expand Down
101 changes: 11 additions & 90 deletions crates/nokv-meta/src/workspace/publish_operation_records.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,14 @@ use nokv_types::{
WorkspaceRevision, FIXED_ID_BYTES, SHA256_BYTES,
};

/// Durable value format for publication-owned payloads.
/// Durable value format for publication-owned payloads. Clients seal manifest
/// rows against this encoding, so it moves only with the manifest row.
pub const PUBLISH_VALUE_FORMAT_VERSION: u8 = 7;
/// Durable value format for the staged-object record. Version 8 dropped the
/// multipart upload id that no provider path ever set; a version 7 record is
/// refused rather than read with the field's bytes shifted into its
/// neighbours.
pub const STAGED_OBJECT_VALUE_FORMAT_VERSION: u8 = 8;
/// Durable value format for the publish operation record. Version 9 records
/// the initiating principal; version 8 added the sealed digest of a body
/// published inline.
Expand All @@ -36,7 +42,6 @@ pub const MAX_DEPENDENCY_DEPTH: u8 = 8;
/// Maximum append-segment sequence admitted by the durable publication codec.
pub const MAX_APPEND_SEGMENTS: u32 = 1_048_576;

pub const MAX_MULTIPART_ID_BYTES: usize = 1_024;
pub const MAX_DIGEST_URI_BYTES: usize = 256;
pub const MAX_TERMINAL_ERROR_BYTES: usize = 4_096;

Expand Down Expand Up @@ -351,7 +356,6 @@ pub struct StagedObjectRecord {
pub artifact_revision_id: ArtifactRevisionId,
pub object_sequence: u32,
pub object_key: String,
pub multipart_upload_id: Option<Vec<u8>>,
pub expected_length: u64,
pub expected_digest_uri: String,
pub provider_state: StagedProviderState,
Expand Down Expand Up @@ -1300,9 +1304,6 @@ impl StagedObjectRecord {
});
}
validate_object_key(&self.object_key, self.artifact_revision_id)?;
if let Some(upload_id) = &self.multipart_upload_id {
validate_opaque_bytes("multipart_upload_id", upload_id, MAX_MULTIPART_ID_BYTES)?;
}
if self.expected_length == 0 {
return Err(PublishRecordError::ZeroScalar {
field: "expected_length",
Expand All @@ -1315,15 +1316,10 @@ impl StagedObjectRecord {
pub fn encode(&self) -> Result<Vec<u8>, PublishRecordError> {
self.validate()?;
let mut encoded = Vec::with_capacity(256);
encoded.push(PUBLISH_VALUE_FORMAT_VERSION);
encoded.push(STAGED_OBJECT_VALUE_FORMAT_VERSION);
encoded.extend_from_slice(self.artifact_revision_id.as_bytes());
encoded.extend_from_slice(&self.object_sequence.to_be_bytes());
push_string(&mut encoded, "object_key", &self.object_key)?;
push_optional_bytes(
&mut encoded,
"multipart_upload_id",
self.multipart_upload_id.as_deref(),
)?;
encoded.extend_from_slice(&self.expected_length.to_be_bytes());
push_string(
&mut encoded,
Expand All @@ -1347,13 +1343,11 @@ impl StagedObjectRecord {
MAX_STAGED_OBJECT_RECORD_BYTES,
)?;
let mut decoder = Decoder::new(encoded);
decoder.require_value_version()?;
decoder.require_version(STAGED_OBJECT_VALUE_FORMAT_VERSION)?;
let artifact_revision_id =
ArtifactRevisionId::from_bytes(decoder.fixed("artifact_revision_id")?);
let object_sequence = decoder.u32("object_sequence")?;
let object_key = decoder.string("object_key", OBJECT_KEY_BYTES)?;
let multipart_upload_id =
decoder.optional_bytes("multipart_upload_id", MAX_MULTIPART_ID_BYTES)?;
let expected_length = decoder.u64("expected_length")?;
let expected_digest_uri = decoder.string("expected_digest_uri", MAX_DIGEST_URI_BYTES)?;
let provider_state = decode_provider_state(decoder.u8("provider_state")?)?;
Expand All @@ -1363,7 +1357,6 @@ impl StagedObjectRecord {
artifact_revision_id,
object_sequence,
object_key,
multipart_upload_id,
expected_length,
expected_digest_uri,
provider_state,
Expand Down Expand Up @@ -1576,13 +1569,10 @@ fn validate_staged_state_pair(
let valid = matches!(
(provider, cleanup),
(
StagedProviderState::Planned
| StagedProviderState::Uploading
| StagedProviderState::Uploaded,
StagedProviderState::Planned | StagedProviderState::Uploaded,
StagedCleanupState::Owned
) | (
StagedProviderState::Planned
| StagedProviderState::Uploading
| StagedProviderState::Uploaded
| StagedProviderState::AbortPending,
StagedCleanupState::DeletePending
Expand Down Expand Up @@ -1692,24 +1682,6 @@ fn validate_text(field: &'static str, value: &str, max: usize) -> Result<(), Pub
Ok(())
}

fn validate_opaque_bytes(
field: &'static str,
value: &[u8],
max: usize,
) -> Result<(), PublishRecordError> {
if value.is_empty() {
return Err(PublishRecordError::EmptyField { field });
}
if value.len() > max {
return Err(PublishRecordError::FieldTooLong {
field,
length: value.len(),
max,
});
}
Ok(())
}

fn require_record_size(
record: &'static str,
length: usize,
Expand Down Expand Up @@ -1880,21 +1852,6 @@ fn push_bytes(
Ok(())
}

fn push_optional_bytes(
target: &mut Vec<u8>,
field: &'static str,
value: Option<&[u8]>,
) -> Result<(), PublishRecordError> {
match value {
None => target.push(0),
Some(value) => {
target.push(1);
push_bytes(target, field, value)?;
}
}
Ok(())
}

fn push_optional_fixed<const N: usize>(target: &mut Vec<u8>, value: &Option<[u8; N]>) {
match value {
None => target.push(0),
Expand Down Expand Up @@ -1966,18 +1923,6 @@ impl<'a> Decoder<'a> {
.map_err(|_| PublishRecordError::InvalidUtf8 { field })
}

fn optional_bytes(
&mut self,
field: &'static str,
max: usize,
) -> Result<Option<Vec<u8>>, PublishRecordError> {
match self.u8(field)? {
0 => Ok(None),
1 => self.bytes(field, max).map(Some),
value => Err(PublishRecordError::InvalidOptionalTag { field, value }),
}
}

fn optional_fixed<const N: usize>(
&mut self,
field: &'static str,
Expand Down Expand Up @@ -2288,22 +2233,18 @@ mod tests {
artifact_revision_id: revision(0x22),
object_sequence: 7,
object_key: object_key(0x22),
multipart_upload_id: Some(b"upload-1".to_vec()),
expected_length: 4_096,
expected_digest_uri: "sha256:block".to_owned(),
provider_state: StagedProviderState::Uploaded,
cleanup_state: StagedCleanupState::Owned,
};
let key = object_key(0x22);
let expected = [
&[PUBLISH_VALUE_FORMAT_VERSION][..],
&[STAGED_OBJECT_VALUE_FORMAT_VERSION][..],
&[0x22; 16],
&7_u32.to_be_bytes(),
&(key.len() as u32).to_be_bytes(),
key.as_bytes(),
&[1],
&8_u32.to_be_bytes(),
b"upload-1",
&4_096_u64.to_be_bytes(),
&12_u32.to_be_bytes(),
b"sha256:block",
Expand Down Expand Up @@ -2649,7 +2590,6 @@ mod tests {
artifact_revision_id: revision(0x22),
object_sequence: 0,
object_key: object_key(0x22),
multipart_upload_id: None,
expected_length: 1,
expected_digest_uri: "sha256:a".to_owned(),
provider_state: StagedProviderState::Uploaded,
Expand Down Expand Up @@ -2760,7 +2700,6 @@ mod tests {
artifact_revision_id: revision(0x22),
object_sequence: 0,
object_key: object_key(0x22),
multipart_upload_id: None,
expected_length: 1,
expected_digest_uri: "sha256:a".to_owned(),
provider_state: StagedProviderState::Uploaded,
Expand Down Expand Up @@ -2857,24 +2796,6 @@ mod tests {

#[test]
fn oversized_fields_and_records_fail_before_allocation_or_acceptance() {
let staged = StagedObjectRecord {
artifact_revision_id: revision(0x22),
object_sequence: 0,
object_key: object_key(0x22),
multipart_upload_id: Some(vec![0; MAX_MULTIPART_ID_BYTES + 1]),
expected_length: 1,
expected_digest_uri: "sha256:a".to_owned(),
provider_state: StagedProviderState::Uploading,
cleanup_state: StagedCleanupState::Owned,
};
assert!(matches!(
staged.encode(),
Err(PublishRecordError::FieldTooLong {
field: "multipart_upload_id",
..
})
));

let oversized = vec![0; MAX_PUBLISH_OPERATION_RECORD_BYTES + 1];
assert_eq!(
PublishOperationRecord::decode(&oversized).unwrap_err(),
Expand Down
2 changes: 0 additions & 2 deletions crates/nokv-meta/src/workspace/restore.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10794,7 +10794,6 @@ mod tests {
artifact_revision_id,
object_sequence: 0,
object_key: object_block_key(shard(), root(), artifact_revision_id, 0),
multipart_upload_id: None,
expected_length: body.len() as u64,
expected_digest_uri: body_digest_uri(body),
provider_state: StagedProviderState::Planned,
Expand Down Expand Up @@ -10836,7 +10835,6 @@ mod tests {
artifact_revision_id,
object_sequence: object_index,
object_key: object_key.clone(),
multipart_upload_id: None,
expected_length: block_size,
expected_digest_uri: digest_uri.clone(),
provider_state: StagedProviderState::Planned,
Expand Down
1 change: 0 additions & 1 deletion crates/nokv-protocol/src/artifact_publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,6 @@ mod tests {
object_identity: ObjectIdentity::new(key).unwrap(),
expected_length: 4,
expected_digest: digest("sha256:abcd"),
multipart_token: None,
}
}

Expand Down
5 changes: 2 additions & 3 deletions crates/nokv-protocol/src/codec.rs
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,6 @@ mod tests {
object_identity: object(),
expected_length: 1,
expected_digest: digest.clone(),
multipart_token: None,
}],
},
),
Expand Down Expand Up @@ -445,8 +444,8 @@ mod tests {
assert_eq!(
<[u8; 32]>::from(golden.finalize()),
[
95, 157, 43, 219, 156, 245, 126, 182, 88, 13, 148, 128, 96, 253, 190, 238, 237,
127, 173, 41, 108, 173, 150, 72, 236, 15, 70, 203, 2, 183, 100, 149,
108, 207, 191, 121, 68, 172, 63, 22, 194, 9, 128, 128, 137, 110, 101, 239, 11, 98,
201, 199, 14, 192, 40, 140, 37, 233, 60, 50, 125, 247, 177, 184,
],
"update only for an intentional fused publication v15 wire change"
);
Expand Down
1 change: 0 additions & 1 deletion crates/nokv-protocol/src/request.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2373,7 +2373,6 @@ mod tests {
object_identity: crate::ObjectIdentity::new(format!("objects/{sequence}")).unwrap(),
expected_length: 1,
expected_digest: crate::sha256_digest_uri(Digest([5; 32])),
multipart_token: None,
}
}

Expand Down
13 changes: 0 additions & 13 deletions crates/nokv-protocol/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -539,29 +539,16 @@ pub struct StagedObject {
pub object_identity: ObjectIdentity,
pub expected_length: u64,
pub expected_digest: DigestUri,
pub multipart_token: Option<Vec<u8>>,
}

impl StagedObject {
pub const MAX_MULTIPART_TOKEN_BYTES: usize = 4_096;

pub fn validate(&self) -> Result<(), ProtocolError> {
if self.expected_length == 0 {
return Err(ProtocolError::invalid(
"staged_object.expected_length",
"must be greater than zero",
));
}
if self
.multipart_token
.as_ref()
.is_some_and(|token| token.len() > Self::MAX_MULTIPART_TOKEN_BYTES)
{
return Err(ProtocolError::invalid(
"staged_object.multipart_token",
format!("exceeds {} bytes", Self::MAX_MULTIPART_TOKEN_BYTES),
));
}
Ok(())
}
}
Expand Down
4 changes: 2 additions & 2 deletions crates/nokv-python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,8 @@ not for installing the SDK.
same status plus logical `operation_token`, child `publication_token`, object
namespace, `registered_count`, `cleanup_cursor`, `remaining_count`, and a page
of retained staged `entries`. Each entry has `sequence`, `object_identity`,
`expected_length`, `expected_digest`, and optional `multipart_token`. Retired
keys are not a historical inventory. The limit is 1 to 192; `next_cursor` is
`expected_length`, and `expected_digest`. Retired keys are not a historical
inventory. The limit is 1 to 192; `next_cursor` is
opaque `bytes` or `None`. Continue with the unchanged cursor. A changed state
returns `Conflict`; restart inspection rather than combining different pages.
- `Client.operation_recover(operation_id, expected_state_digest=None)` asks the
Expand Down
Loading
Loading