diff --git a/docs/serialization.md b/docs/serialization.md index d756211..c40ee6d 100644 --- a/docs/serialization.md +++ b/docs/serialization.md @@ -22,6 +22,8 @@ user = User.from_binary(data) ### Options +#### Unknown Fields + When a message is parsed from binary data containing field numbers it doesn't recognize, the unknown fields are stored internally and re-emitted during serialization. This means a message can pass through an intermediary that doesn't know about newer fields without losing data. @@ -37,6 +39,31 @@ data = user.to_binary(write_unknown_fields=False) user = User.from_binary(data, ignore_unknown_fields=True) ``` +#### Allocation Limit + +Because protobuf serialization can create very compact binary payloads, it is possible for the memory usage of a +parsed message to differ drastically from the input number of bytes. When parsing untrusted payloads, +such as in an external-facing API server, this can allow malicious users to send small messages that take +a large amount of memory or potentially crashing the server. This is most pronounced in schemas with +repeated fields of message type with a large number of fields. + +You can mitigate this using the `allocation_limit` option in `from_binary`. When set, an estimate of the +memory usage of a message is maintained while it is parsed, and if it goes over the limit, the parse fails +immediately before processing the entire payload. The limit is based on the schema and content, not the environment. +Some fixed overhead of objects such as a string can change between Python versions, so the allocation budget should not be +considered a precise value. Such changes are generally relatively small and fixed so an allocation limit determined +for one environment should mostly work fine when i.e., updating Python. + +It is recommended to set an allocation limit for applications parsing untrusted payloads based on your target +memory usage. You may need to experiment with values to see the effective memory utilization corresponding to +the limit. + +```python +user = User.from_binary( + data, allocation_limit=8 * 1024 * 1024 +) # Roughly cap memory usage of parsed message to 8MB +``` + ## JSON ```python diff --git a/packages/protobuf-py-ext/src/budget.rs b/packages/protobuf-py-ext/src/budget.rs new file mode 100644 index 0000000..4ce7368 --- /dev/null +++ b/packages/protobuf-py-ext/src/budget.rs @@ -0,0 +1,242 @@ +use pyo3::{Bound, PyResult, exceptions::PyValueError, types::PyString}; + +use crate::marshaler::MessageMarshaler; + +// Approximate sizes of CPython heap allocations, measured with +// `sys.getsizeof` on 64-bit CPython 3.14. The budget guards against +// unbounded allocation from malicious payloads rather than providing exact +// accounting, so small inaccuracies across versions and builds are fine. + +/// GC header allocated in front of every GC-tracked object. +const GC_HEAD_SIZE: usize = 16; +/// A float object. +const FLOAT_SIZE: usize = 24; +/// A 64-bit int object. Smaller ints are slightly smaller. +const INT_SIZE: usize = 36; +/// Header of a bytes object. +const BYTES_OVERHEAD: usize = 33; +/// An empty list. +const EMPTY_LIST_SIZE: usize = 56; +/// An empty dict. +const EMPTY_DICT_SIZE: usize = 64; +/// One appended list element: an 8-byte pointer slot. +const LIST_SLOT_SIZE: usize = 8; +/// One inserted dict entry: hash + key + value words plus growth slack. +const DICT_ENTRY_SIZE: usize = 40; +/// A `Oneof` wrapper object: `PyObject` header plus two object pointers. +const ONEOF_SIZE: usize = 32; + +// Python strings have two structs, one for ASCII and one for others. +// They also always contain a trailing NUL, so the total overhead is +// the size of the header (40 or 56) + the size of the trailing NUL +// which depends on the width. +const STR_ASCII_OVERHEAD: usize = 41; +const STR_LATIN1_OVERHEAD: usize = 57; +const STR_UCS2_OVERHEAD: usize = 58; +const STR_UCS4_OVERHEAD: usize = 60; + +/// The size of a str holding `chars` characters and its largest codepoint. +/// Python picks different representations based on the largest codepoint. +fn str_size_for(chars: usize, max_char: u32) -> usize { + match max_char { + 0..0x80 => STR_ASCII_OVERHEAD + chars, + 0x80..0x100 => STR_LATIN1_OVERHEAD + chars, + 0x100..0x1_0000 => STR_UCS2_OVERHEAD + 2 * chars, + _ => STR_UCS4_OVERHEAD + 4 * chars, + } +} + +/// The decoded size of a str. +fn str_size(utf8: &[u8]) -> usize { + let max_byte = utf8.iter().fold(0, |max, &byte| max.max(byte)); + if max_byte < 0x80 { + return STR_ASCII_OVERHEAD + utf8.len(); + } + let chars = utf8.iter().filter(|&&byte| byte & 0xC0 != 0x80).count(); + match max_byte { + // U+0080 to U+00FF + 0..=0xC3 => STR_LATIN1_OVERHEAD + chars, + // U+0100 to U+FFFF + 0xC4..=0xEF => STR_UCS2_OVERHEAD + 2 * chars, + _ => STR_UCS4_OVERHEAD + 4 * chars, + } +} + +/// The size of Python str. +#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] +fn py_str_size(value: &Bound<'_, PyString>) -> PyResult { + use pyo3::types::{PyStringMethods as _, string::PyStringData}; + + // SAFETY - the data is only read, while the str is borrowed. + Ok(match unsafe { value.data()? } { + // An ASCII str has a smaller header than other single byte ones. + PyStringData::Ucs1(data) => { + let max_char = data.iter().fold(0, |max, &ch| max.max(ch)); + str_size_for(data.len(), max_char.into()) + } + PyStringData::Ucs2(data) => STR_UCS2_OVERHEAD + 2 * data.len(), + PyStringData::Ucs4(data) => STR_UCS4_OVERHEAD + 4 * data.len(), + }) +} + +/// The size of Python str. The limited API cannot read how a str is stored, +/// so this finds its widest character one at a time. +#[cfg(any(Py_LIMITED_API, PyPy, GraalPy))] +fn py_str_size(value: &Bound<'_, PyString>) -> PyResult { + use pyo3::{ffi, types::PyAnyMethods as _}; + + let len = value.len()?; + let mut max_char = 0; + for i in 0..len { + // SAFETY - value is a str and i is within its length. + let ch = unsafe { ffi::PyUnicode_ReadChar(value.as_ptr(), i.cast_signed()) }; + max_char = max_char.max(ch); + if max_char > 0xFFFF { + break; + } + } + Ok(str_size_for(len, max_char)) +} + +/// The size of a new message whose type has the given `__basicsize__` and +/// numbers of repeated and map fields, which start as empty containers. +pub(crate) fn message_size(basic_size: usize, lists: usize, dicts: usize) -> usize { + basic_size + GC_HEAD_SIZE + lists * EMPTY_LIST_SIZE + dicts * EMPTY_DICT_SIZE +} + +/// Accounts for the Python objects allocated while parsing a message. +pub(crate) trait Budget { + /// Charges `amount` bytes. + fn charge(&mut self, amount: usize) -> PyResult<()>; + + /// Charges for the pre-decoded str. + fn charge_str(&mut self, utf8: &[u8]) -> PyResult<()>; + + /// Charges for an existing Python string. + fn charge_py_str(&mut self, value: &Bound<'_, PyString>) -> PyResult<()>; + + /// Charges for a new message of the marshaler's type. + fn charge_message(&mut self, marshaler: &MessageMarshaler) -> PyResult<()> { + self.charge(marshaler.alloc_size) + } + + /// Charges for an int. Small ints are shared and larger ones vary a + /// little in size, but a flat charge is close enough. + fn charge_int(&mut self) -> PyResult<()> { + self.charge(INT_SIZE) + } + + /// Charges for a float. + fn charge_float(&mut self) -> PyResult<()> { + self.charge(FLOAT_SIZE) + } + + /// Charges for a bytes of the given length. + fn charge_bytes(&mut self, len: usize) -> PyResult<()> { + self.charge(BYTES_OVERHEAD + len) + } + + /// Charges for a materialized unknown enum member. + fn charge_enum(&mut self) -> PyResult<()> { + self.charge(INT_SIZE + GC_HEAD_SIZE) + } + + /// Charges for a `Oneof` wrapper. + fn charge_oneof(&mut self) -> PyResult<()> { + self.charge(ONEOF_SIZE) + } + + /// Charges for a new list. + fn charge_list(&mut self) -> PyResult<()> { + self.charge(EMPTY_LIST_SIZE) + } + + /// Charges for appending an item to a list. + fn charge_list_item(&mut self) -> PyResult<()> { + self.charge(LIST_SLOT_SIZE) + } + + /// Charges for a new dict. + fn charge_dict(&mut self) -> PyResult<()> { + self.charge(EMPTY_DICT_SIZE) + } + + /// Charges for inserting an entry into a dict. + fn charge_dict_entry(&mut self) -> PyResult<()> { + self.charge(DICT_ENTRY_SIZE) + } +} + +/// Tracks the approximate bytes allocated, raising an error once a limit is +/// exceeded. +pub(crate) struct TrackingBudget { + current: usize, + max: usize, +} + +impl TrackingBudget { + pub(crate) fn new(limit: usize) -> Self { + Self { + current: 0, + max: limit, + } + } +} + +impl Budget for TrackingBudget { + fn charge(&mut self, amount: usize) -> PyResult<()> { + self.current = self.current.saturating_add(amount); + if self.current > self.max { + Err(PyValueError::new_err(format!( + "allocation budget exceeded: needed {} bytes, limit is {}", + self.current, self.max + ))) + } else { + Ok(()) + } + } + + fn charge_str(&mut self, utf8: &[u8]) -> PyResult<()> { + self.charge(str_size(utf8)) + } + + fn charge_py_str(&mut self, value: &Bound<'_, PyString>) -> PyResult<()> { + self.charge(py_str_size(value)?) + } +} + +/// Does no accounting, for parsing without an allocation limit. +pub(crate) struct NoopBudget; + +impl Budget for NoopBudget { + fn charge(&mut self, _amount: usize) -> PyResult<()> { + Ok(()) + } + + fn charge_str(&mut self, _utf8: &[u8]) -> PyResult<()> { + Ok(()) + } + + fn charge_py_str(&mut self, _value: &Bound<'_, PyString>) -> PyResult<()> { + Ok(()) + } +} + +/// Evaluates `$body` with `$budget` bound to the budget for an optional +/// allocation limit. The body is compiled for each implementation, so parsing +/// without a limit includes no accounting code. +macro_rules! with_budget { + ($limit:expr, |$budget:ident| $body:expr) => { + match $limit { + Some(limit) => { + let $budget = &mut $crate::budget::TrackingBudget::new(limit); + $body + } + None => { + let $budget = &mut $crate::budget::NoopBudget; + $body + } + } + }; +} +pub(crate) use with_budget; diff --git a/packages/protobuf-py-ext/src/constants.rs b/packages/protobuf-py-ext/src/constants.rs index 229bd32..7ccbfff 100644 --- a/packages/protobuf-py-ext/src/constants.rs +++ b/packages/protobuf-py-ext/src/constants.rs @@ -93,6 +93,8 @@ pub(crate) struct ConstantsInner { /// The string `__new__`. pub(crate) dunder_new: Py, + /// The string `__basicsize__`. + pub(crate) dunder_basicsize: Py, /// Python types. pub(crate) types: Types, @@ -146,6 +148,7 @@ impl Constants { values: PyString::new(py, "values").unbind(), dunder_new: PyString::new(py, "__new__").unbind(), + dunder_basicsize: PyString::new(py, "__basicsize__").unbind(), types: Types { desc_field: mod_descriptors diff --git a/packages/protobuf-py-ext/src/json_parse.rs b/packages/protobuf-py-ext/src/json_parse.rs index 81a900f..bfc92aa 100644 --- a/packages/protobuf-py-ext/src/json_parse.rs +++ b/packages/protobuf-py-ext/src/json_parse.rs @@ -17,6 +17,7 @@ use pyo3::{ }; use crate::{ + budget::Budget, descriptor::{DescEnum, DescFieldValue, ScalarType}, json_source::{JiterSource, JsonKind, JsonSource, PyTreeSource}, marshaler::MessageMarshaler, @@ -53,9 +54,10 @@ pub(crate) fn merge_from_json<'py>( message: &Bound<'py, NativeMessage>, data: &[u8], opts: &FromJsonOpts, + budget: &mut impl Budget, ) -> PyResult<()> { let mut src = JiterSource::new(py, data); - read_message(marshaler, message, &mut src, opts, 0)?; + read_message(marshaler, message, &mut src, opts, 0, budget)?; src.finish() } @@ -66,9 +68,10 @@ pub(crate) fn read_message_from_tree<'py>( message: &Bound<'py, NativeMessage>, tree: Bound<'py, PyAny>, opts: &FromJsonOpts, + budget: &mut impl Budget, ) -> PyResult<()> { let mut src = PyTreeSource::new(py, tree); - read_message(marshaler, message, &mut src, opts, 0) + read_message(marshaler, message, &mut src, opts, 0, budget) } /// Reads a message value, dispatching on its well-known-type kind. Ordinary @@ -80,6 +83,7 @@ pub(crate) fn read_message<'py, R: JsonSource<'py>>( src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { if depth > DEPTH_LIMIT { return Err(pyo3::exceptions::PyRecursionError::new_err(format!( @@ -87,8 +91,8 @@ pub(crate) fn read_message<'py, R: JsonSource<'py>>( ))); } match &marshaler.wkt { - Some(wkt) => wkt.read_json(marshaler, message, src, opts, depth), - None => read_generic_object(marshaler, message, src, opts, depth), + Some(wkt) => wkt.read_json(marshaler, message, src, opts, depth, budget), + None => read_generic_object(marshaler, message, src, opts, depth, budget), } } @@ -98,10 +102,11 @@ fn read_generic_object<'py, R: JsonSource<'py>>( src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); if src.peek()? != JsonKind::Object { - let value = read_json_value(src)?; + let value = read_json_value(src, budget)?; let qualname = marshaler.python_type.bind(py).qualname()?; return Err(PyTypeError::new_err(format!( "cannot decode {qualname} from JSON: {}", @@ -132,9 +137,9 @@ fn read_generic_object<'py, R: JsonSource<'py>>( if merges_on_duplicate(parser) && !seen.insert(number) { reset_duplicate_field(py, parser, message)?; } - read_field(marshaler, parser, message, src, opts, depth)?; + read_field(marshaler, parser, message, src, opts, depth, budget)?; } else { - handle_unknown_key(marshaler, message, key, src, opts, depth)?; + handle_unknown_key(marshaler, message, key, src, opts, depth, budget)?; } Ok(()) })?; @@ -178,6 +183,7 @@ fn read_field<'py, R: JsonSource<'py>>( src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { match &parser.type_ { ParserFieldType::Singular { @@ -192,8 +198,11 @@ fn read_field<'py, R: JsonSource<'py>>( depth, oneof_attr.as_ref(), *requires_presence, + budget, ), - ParserFieldType::List { .. } => read_list(marshaler, parser, message, src, opts, depth), + ParserFieldType::List { .. } => { + read_list(marshaler, parser, message, src, opts, depth, budget) + } ParserFieldType::Map { key_type, value_parser, @@ -207,6 +216,7 @@ fn read_field<'py, R: JsonSource<'py>>( depth, *key_type, value_parser, + budget, ), } } @@ -221,6 +231,7 @@ fn read_singular<'py, R: JsonSource<'py>>( depth: usize, oneof_attr: Option<&crate::attribute_access::AttributeAccess>, requires_presence: bool, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); match &parser.value { @@ -238,7 +249,10 @@ fn read_singular<'py, R: JsonSource<'py>>( } let name = parser.name.bind(py); let ctx = FieldContext::Field { marshaler, name }; - let value = read_scalar(&ctx, src, *scalar)?; + let value = read_scalar(&ctx, src, *scalar, budget)?; + if oneof_attr.is_some() { + budget.charge_oneof()?; + } parser.assign_singular(py, message, &value, oneof_attr, requires_presence) } FieldParserValue::Enum(enum_) => { @@ -253,7 +267,10 @@ fn read_singular<'py, R: JsonSource<'py>>( )?; return Ok(()); } - if let Some(value) = read_enum(enum_, src, opts)? { + if let Some(value) = read_enum(enum_, src, opts, budget)? { + if oneof_attr.is_some() { + budget.charge_oneof()?; + } parser.assign_singular(py, message, &value, oneof_attr, requires_presence)?; } Ok(()) @@ -271,9 +288,15 @@ fn read_singular<'py, R: JsonSource<'py>>( let existing = parser.get_field_value(py, message)?; let target = match existing { Some(value) if !value.is_none() => value.cast_into::()?, - _ => inner.new_empty_message(py, msg_desc.get_python_type(py))?, + _ => { + budget.charge_message(inner)?; + inner.new_empty_message(py, msg_desc.get_python_type(py))? + } }; - read_message(inner, &target, src, opts, depth + 1)?; + read_message(inner, &target, src, opts, depth + 1, budget)?; + if oneof_attr.is_some() { + budget.charge_oneof()?; + } parser.assign_singular(py, message, target.as_any(), oneof_attr, requires_presence) } } @@ -312,6 +335,7 @@ fn read_list<'py, R: JsonSource<'py>>( src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); let name = parser.name.bind(py); @@ -322,7 +346,10 @@ fn read_list<'py, R: JsonSource<'py>>( } if src.peek()? != JsonKind::Array { return Err(ctx.error( - &format!("expected list got {}", read_json_value(src)?.get_type()), + &format!( + "expected list got {}", + read_json_value(src, budget)?.get_type() + ), Exc::Type, )); } @@ -331,7 +358,10 @@ fn read_list<'py, R: JsonSource<'py>>( .get(py, message.as_any())? .cast_into::()?; src.for_each_array_item(|src| { - if let Some(value) = read_container_item(&ctx, &parser.value, src, opts, depth, false)? { + if let Some(value) = + read_container_item(&ctx, &parser.value, src, opts, depth, false, budget)? + { + budget.charge_list_item()?; list.append(value)?; } Ok(()) @@ -349,6 +379,7 @@ fn read_map<'py, R: JsonSource<'py>>( depth: usize, key_type: ScalarType, value_parser: &FieldParser, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); let name = parser.name.bind(py); @@ -359,7 +390,10 @@ fn read_map<'py, R: JsonSource<'py>>( } if src.peek()? != JsonKind::Object { return Err(ctx.error( - &format!("expected dict got {}", read_json_value(src)?.get_type()), + &format!( + "expected dict got {}", + read_json_value(src, budget)?.get_type() + ), Exc::Type, )); } @@ -368,9 +402,11 @@ fn read_map<'py, R: JsonSource<'py>>( .get(py, message.as_any())? .cast_into::()?; src.for_each_object_key(|key, src| { - let map_key = read_map_key(py, &ctx, key_type, key)?; - if let Some(value) = read_container_item(&ctx, &value_parser.value, src, opts, depth, true)? + let map_key = read_map_key(py, &ctx, key_type, key, budget)?; + if let Some(value) = + read_container_item(&ctx, &value_parser.value, src, opts, depth, true, budget)? { + budget.charge_dict_entry()?; dict.set_item(map_key, value)?; } Ok(()) @@ -387,11 +423,14 @@ fn read_container_item<'py, R: JsonSource<'py>>( opts: &FromJsonOpts, depth: usize, is_map: bool, + budget: &mut impl Budget, ) -> PyResult>> { let py = src.py(); let is_null = src.peek()? == JsonKind::Null; match element { - FieldParserValue::Scalar(scalar) if !is_null => Ok(Some(read_scalar(ctx, src, *scalar)?)), + FieldParserValue::Scalar(scalar) if !is_null => { + Ok(Some(read_scalar(ctx, src, *scalar, budget)?)) + } FieldParserValue::Message { message: msg_desc, .. } => { @@ -401,12 +440,13 @@ fn read_container_item<'py, R: JsonSource<'py>>( src.next_null()?; return Err(container_null_error(ctx, is_map)); } + budget.charge_message(inner)?; let target = inner.new_empty_message(py, msg_desc.get_python_type(py))?; - read_message(inner, &target, src, opts, depth + 1)?; + read_message(inner, &target, src, opts, depth + 1, budget)?; Ok(Some(target.into_any())) } FieldParserValue::Enum(enum_) if !is_null || enum_.is_null_value => { - read_enum(enum_, src, opts) + read_enum(enum_, src, opts, budget) } _ => { // Resetting null for a list item / map value: error. @@ -426,6 +466,7 @@ fn read_map_key<'py>( ctx: &FieldContext<'_, 'py>, key_type: ScalarType, raw_key: &str, + budget: &mut impl Budget, ) -> PyResult> { match key_type { ScalarType::Bool => match raw_key { @@ -436,8 +477,14 @@ fn read_map_key<'py>( Exc::Value, )), }, - ScalarType::String => Ok(PyString::new(py, raw_key).into_any()), - _ => parse_int_string(py, ctx, raw_key, key_type), + ScalarType::String => { + budget.charge_str(raw_key.as_bytes())?; + Ok(PyString::new(py, raw_key).into_any()) + } + _ => { + budget.charge_int()?; + parse_int_string(py, ctx, raw_key, key_type) + } } } @@ -446,12 +493,13 @@ pub(crate) fn read_scalar<'py, R: JsonSource<'py>>( ctx: &FieldContext<'_, 'py>, src: &mut R, scalar: ScalarType, + budget: &mut impl Budget, ) -> PyResult> { let py = src.py(); match scalar { ScalarType::Bool => { if src.peek()? != JsonKind::Bool { - let value = read_json_value(src)?; + let value = read_json_value(src, budget)?; return Err(ctx.error( &format!("unexpected json type: {}", value.get_type()), Exc::Type, @@ -460,40 +508,50 @@ pub(crate) fn read_scalar<'py, R: JsonSource<'py>>( Ok(PyBool::new(py, src.next_bool()?).to_owned().into_any()) } ScalarType::Float => { - let value = parse_float(ctx, src)?; + let value = parse_float(ctx, src, budget)?; if value.is_finite() && !(FLOAT32_MIN..=FLOAT32_MAX).contains(&value) { return Err(ctx.error(&format!("float value out of range: {value}"), Exc::Overflow)); } + budget.charge_float()?; Ok(PyFloat::new(py, value).into_any()) } - ScalarType::Double => Ok(PyFloat::new(py, parse_float(ctx, src)?).into_any()), - ScalarType::String => Ok(read_string(ctx, src)?.into_any()), - ScalarType::Bytes => read_bytes(ctx, src), - _ => read_int(ctx, src, scalar), + ScalarType::Double => { + let value = parse_float(ctx, src, budget)?; + budget.charge_float()?; + Ok(PyFloat::new(py, value).into_any()) + } + ScalarType::String => Ok(read_string(ctx, src, budget)?.into_any()), + ScalarType::Bytes => read_bytes(ctx, src, budget), + _ => { + budget.charge_int()?; + read_int(ctx, src, scalar, budget) + } } } fn read_string<'py, R: JsonSource<'py>>( ctx: &FieldContext<'_, 'py>, src: &mut R, + budget: &mut impl Budget, ) -> PyResult> { if src.peek()? != JsonKind::String { - let value = read_json_value(src)?; + let value = read_json_value(src, budget)?; return Err(ctx.error( &format!("expected string got: {}", value.get_type()), Exc::Type, )); } - src.next_py_str() + src.next_py_str(budget) } fn read_bytes<'py, R: JsonSource<'py>>( ctx: &FieldContext<'_, 'py>, src: &mut R, + budget: &mut impl Budget, ) -> PyResult> { let py = src.py(); if src.peek()? != JsonKind::String { - let value = read_json_value(src)?; + let value = read_json_value(src, budget)?; return Err(ctx.error( &format!("expected base64-encoded string got: {}", value.get_type()), Exc::Type, @@ -511,6 +569,7 @@ fn read_bytes<'py, R: JsonSource<'py>>( .map_err(|_| ctx.error("invalid base64 data", Exc::Value))?; Ok(decoded) })?; + budget.charge_bytes(decoded.len())?; Ok(PyBytes::new(py, &decoded).into_any()) } @@ -531,7 +590,11 @@ fn base64_url_safe() -> base64::engine::GeneralPurpose { } /// Parses a float/double. -fn parse_float<'py, R: JsonSource<'py>>(ctx: &FieldContext<'_, 'py>, src: &mut R) -> PyResult { +fn parse_float<'py, R: JsonSource<'py>>( + ctx: &FieldContext<'_, 'py>, + src: &mut R, + budget: &mut impl Budget, +) -> PyResult { match src.peek()? { JsonKind::Number => { let value = src.next_float()?; @@ -560,7 +623,7 @@ fn parse_float<'py, R: JsonSource<'py>>(ctx: &FieldContext<'_, 'py>, src: &mut R } }), _ => { - let value = read_json_value(src)?; + let value = read_json_value(src, budget)?; Err(ctx.error( &format!("unexpected json type: {}", value.get_type()), Exc::Type, @@ -574,6 +637,7 @@ fn read_int<'py, R: JsonSource<'py>>( ctx: &FieldContext<'_, 'py>, src: &mut R, int_type: ScalarType, + budget: &mut impl Budget, ) -> PyResult> { let py = src.py(); let value = match src.peek()? { @@ -599,7 +663,7 @@ fn read_int<'py, R: JsonSource<'py>>( }); } _ => { - let value = read_json_value(src)?; + let value = read_json_value(src, budget)?; return Err(ctx.error( &format!("unexpected json type: {}", value.get_type()), Exc::Type, @@ -676,6 +740,7 @@ fn read_enum<'py, R: JsonSource<'py>>( enum_desc: &DescEnum, src: &mut R, opts: &FromJsonOpts, + budget: &mut impl Budget, ) -> PyResult>> { let py = src.py(); match src.peek()? { @@ -694,6 +759,7 @@ fn read_enum<'py, R: JsonSource<'py>>( if opts.ignore_unknown_fields { return Ok(None); } + budget.charge_enum()?; return Ok(Some(enum_desc.py_type.bind(py).call1((number,))?)); }; if let Some(value) = enum_desc.values.get(&int_value) { @@ -702,6 +768,7 @@ fn read_enum<'py, R: JsonSource<'py>>( Ok(None) } else { // Open enum: succeeds; closed enum: raises via Python enum call. + budget.charge_enum()?; Ok(Some(enum_desc.py_type.bind(py).call1((int_value,))?)) } } @@ -720,7 +787,7 @@ fn read_enum<'py, R: JsonSource<'py>>( } }), _ => { - let value = read_json_value(src)?; + let value = read_json_value(src, budget)?; Err(decode_enum_error(py, enum_desc, &value)) } } @@ -745,6 +812,7 @@ fn handle_unknown_key<'py, R: JsonSource<'py>>( src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); if raw_key.starts_with('[') @@ -760,7 +828,7 @@ fn handle_unknown_key<'py, R: JsonSource<'py>>( let extendee_name = extendee.getattr(&marshaler.constants.type_name)?; let extendee_name = extendee_name.cast::()?.to_str()?; if extendee_name == &*marshaler.type_name { - read_extension(marshaler, message, &extension, src, opts, depth)?; + read_extension(marshaler, message, &extension, src, opts, depth, budget)?; } else { src.skip()?; } @@ -785,6 +853,7 @@ fn read_extension<'py, R: JsonSource<'py>>( src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); let ext_type = extension.getattr(&marshaler.constants.type_)?; @@ -803,7 +872,7 @@ fn read_extension<'py, R: JsonSource<'py>>( src.next_null()?; target.del_item(&ext_type)?; } else { - let value = read_scalar(&ctx, src, *scalar_type)?; + let value = read_scalar(&ctx, src, *scalar_type, budget)?; target.set_item(&ext_type, value)?; } } @@ -811,7 +880,7 @@ fn read_extension<'py, R: JsonSource<'py>>( if src.peek()? == JsonKind::Null && !enum_.is_null_value { src.next_null()?; target.del_item(&ext_type)?; - } else if let Some(value) = read_enum(enum_, src, opts)? { + } else if let Some(value) = read_enum(enum_, src, opts, budget)? { target.set_item(&ext_type, value)?; } } @@ -824,8 +893,9 @@ fn read_extension<'py, R: JsonSource<'py>>( src.next_null()?; target.del_item(&ext_type)?; } else { + budget.charge_message(inner)?; let value = inner.new_empty_message(py, msg_desc.get_python_type(py))?; - read_message(inner, &value, src, opts, depth + 1)?; + read_message(inner, &value, src, opts, depth + 1, budget)?; target.set_item(&ext_type, value)?; } } @@ -837,16 +907,21 @@ fn read_extension<'py, R: JsonSource<'py>>( } if src.peek()? != JsonKind::Array { return Err(ctx.error( - &format!("expected list got {}", read_json_value(src)?.get_type()), + &format!( + "expected list got {}", + read_json_value(src, budget)?.get_type() + ), Exc::Type, )); } let element_value = FieldParserValue::from_desc_single(element); + budget.charge_list()?; let list = PyList::empty(py); src.for_each_array_item(|src| { if let Some(value) = - read_container_item(&ctx, &element_value, src, opts, depth, false)? + read_container_item(&ctx, &element_value, src, opts, depth, false, budget)? { + budget.charge_list_item()?; list.append(value)?; } Ok(()) @@ -862,7 +937,10 @@ fn read_extension<'py, R: JsonSource<'py>>( } /// Materializes the next JSON value as a Python object. -pub(crate) fn read_json_value<'py, R: JsonSource<'py>>(src: &mut R) -> PyResult> { +pub(crate) fn read_json_value<'py, R: JsonSource<'py>>( + src: &mut R, + budget: &mut impl Budget, +) -> PyResult> { let py = src.py(); match src.peek()? { JsonKind::Null => { @@ -870,20 +948,29 @@ pub(crate) fn read_json_value<'py, R: JsonSource<'py>>(src: &mut R) -> PyResult< Ok(py.None().into_bound(py)) } JsonKind::Bool => Ok(PyBool::new(py, src.next_bool()?).to_owned().into_any()), - JsonKind::Number => src.next_number(), - JsonKind::String => Ok(src.next_py_str()?.into_any()), + JsonKind::Number => { + budget.charge_int()?; + src.next_number() + } + JsonKind::String => Ok(src.next_py_str(budget)?.into_any()), JsonKind::Array => { + budget.charge_list()?; let list = PyList::empty(py); src.for_each_array_item(|src| { - list.append(read_json_value(src)?)?; + let value = read_json_value(src, budget)?; + budget.charge_list_item()?; + list.append(value)?; Ok(()) })?; Ok(list.into_any()) } JsonKind::Object => { + budget.charge_dict()?; let dict = PyDict::new(py); src.for_each_object_key(|key, src| { - let value = read_json_value(src)?; + let value = read_json_value(src, budget)?; + budget.charge_str(key.as_bytes())?; + budget.charge_dict_entry()?; dict.set_item(PyString::new(py, key), value)?; Ok(()) })?; diff --git a/packages/protobuf-py-ext/src/json_source.rs b/packages/protobuf-py-ext/src/json_source.rs index ca1a8b2..c5f9e05 100644 --- a/packages/protobuf-py-ext/src/json_source.rs +++ b/packages/protobuf-py-ext/src/json_source.rs @@ -9,6 +9,8 @@ use pyo3::{ }, }; +use crate::budget::Budget; + /// The kind of the next JSON value, as reported by `peek`. #[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum JsonKind { @@ -34,8 +36,9 @@ pub(crate) trait JsonSource<'py> { fn next_number(&mut self) -> PyResult>; // Reads a JSON number. fn next_float(&mut self) -> PyResult; - /// Reads a JSON string as a Python string object. - fn next_py_str(&mut self) -> PyResult>; + /// Reads a JSON string as a Python string object, charging the budget + /// for it. + fn next_py_str(&mut self, budget: &mut impl Budget) -> PyResult>; /// Reads a JSON string as a Rust str, calling the function with it. fn with_next_str(&mut self, f: impl FnOnce(&str) -> PyResult) -> PyResult; @@ -163,9 +166,12 @@ impl<'py> JsonSource<'py> for JiterSource<'py> { .map_err(|err| jiter_decode_error(py, data, &err)) } - fn next_py_str(&mut self) -> PyResult> { + fn next_py_str(&mut self, budget: &mut impl Budget) -> PyResult> { let py = self.py; - self.with_next_str(|s| Ok(PyString::new(py, s))) + self.with_next_str(|s| { + budget.charge_str(s.as_bytes())?; + Ok(PyString::new(py, s)) + }) } fn with_next_str(&mut self, f: impl FnOnce(&str) -> PyResult) -> PyResult { @@ -311,13 +317,16 @@ impl<'py> JsonSource<'py> for PyTreeSource<'py> { self.take_current()?.extract::() } - fn next_py_str(&mut self) -> PyResult> { - let value = self.take_current()?; - Ok(value.cast_into::()?) + fn next_py_str(&mut self, budget: &mut impl Budget) -> PyResult> { + let value = self.take_current()?.cast_into::()?; + // The str is shared with the tree rather than allocated, but is + // charged so a message costs the same however its JSON is provided. + budget.charge_py_str(&value)?; + Ok(value) } fn with_next_str(&mut self, f: impl FnOnce(&str) -> PyResult) -> PyResult { - let value = self.next_py_str()?; + let value = self.take_current()?.cast_into::()?; let s = value.to_str()?; f(s) } diff --git a/packages/protobuf-py-ext/src/lib.rs b/packages/protobuf-py-ext/src/lib.rs index 1802da2..ef493f6 100644 --- a/packages/protobuf-py-ext/src/lib.rs +++ b/packages/protobuf-py-ext/src/lib.rs @@ -4,6 +4,7 @@ use pyo3::prelude::*; mod alloc; mod attribute_access; mod bitset; +mod budget; mod buffer; mod constants; mod descriptor; diff --git a/packages/protobuf-py-ext/src/marshaler.rs b/packages/protobuf-py-ext/src/marshaler.rs index e8e7287..57e803f 100644 --- a/packages/protobuf-py-ext/src/marshaler.rs +++ b/packages/protobuf-py-ext/src/marshaler.rs @@ -13,6 +13,7 @@ use pyo3::{ use crate::{ attribute_access::AttributeAccess, + budget::{self, Budget}, constants::Constants, descriptor::{DescFieldValue, message_fields}, nativemessage::NativeMessage, @@ -74,6 +75,11 @@ pub(crate) struct MessageMarshalerInner { /// The maximum field number of the message. pub(crate) max_field_number: u32, + /// Approximate heap size of a freshly-initialized instance of this + /// message type: the fixed instance size (all fields are slots) plus the + /// empty containers created for repeated/map field defaults. + pub(crate) alloc_size: usize, + /// The Python type of the message. pub(crate) python_type: Py, @@ -198,6 +204,14 @@ impl MessageMarshaler { .push((member.attr.clone_ref(py), default.unbind())); } } + // __basicsize__ (tp_basicsize) is the exact instance allocation size: + // message types only use slots, so instances never grow beyond it. The + // GC header is allocated in front of every instance on top of it. + let basic_size = python_type + .getattr(&constants.dunder_basicsize)? + .extract::()?; + let base_alloc_size = + budget::message_size(basic_size, defaults.lists.len(), defaults.dicts.len()); Ok(Self { inner: Arc::new(MessageMarshalerInner { parser, @@ -205,6 +219,7 @@ impl MessageMarshaler { members_by_name: members_by_name.unbind(), members, max_field_number, + alloc_size: base_alloc_size, python_type: python_type.clone().unbind(), type_name, wkt, @@ -222,6 +237,7 @@ impl MessageMarshaler { message: &Bound<'_, NativeMessage>, mut data: Bytes, ignore_unknown_fields: bool, + budget: &mut impl Budget, ) -> PyResult<()> { self.inner.parser.merge_from_binary( py, @@ -231,6 +247,7 @@ impl MessageMarshaler { ignore_unknown_fields, }, 0, + budget, ) } diff --git a/packages/protobuf-py-ext/src/nativemessage.rs b/packages/protobuf-py-ext/src/nativemessage.rs index dd361b3..1eca7ef 100644 --- a/packages/protobuf-py-ext/src/nativemessage.rs +++ b/packages/protobuf-py-ext/src/nativemessage.rs @@ -13,6 +13,7 @@ use pyo3::{ use crate::{ attribute_access::generic_setattr, bitset::BitSet, + budget::{Budget, with_budget}, buffer::Buffer, constants::Constants, json_parse::{FromJsonOpts, merge_from_json, read_message_from_tree}, @@ -100,21 +101,25 @@ impl NativeMessage { } #[classmethod] - #[pyo3(signature = (data, *, ignore_unknown_fields = false))] + #[pyo3(signature = (data, *, ignore_unknown_fields = false, allocation_limit = None))] fn from_binary<'py>( cls: &Bound<'py, PyType>, py: Python<'py>, data: Buffer, ignore_unknown_fields: bool, + allocation_limit: Option, ) -> PyResult> { let data = data.into_inner(); let constants = Constants::get(py)?; let marshaler_any = cls.getattr(&constants.ext_marshaler)?; let marshaler = marshaler_any.cast::()?.get(); - let message = marshaler.new_empty_message(py, cls)?; - let slf = message.cast::()?; - marshaler.merge_from_binary(py, slf, data, ignore_unknown_fields)?; - Ok(message) + with_budget!(allocation_limit, |budget| { + budget.charge_message(marshaler)?; + let message = marshaler.new_empty_message(py, cls)?; + let slf = message.cast::()?; + marshaler.merge_from_binary(py, slf, data, ignore_unknown_fields, budget)?; + Ok(message) + }) } #[pyo3(signature = (*, write_unknown_fields = true))] @@ -146,39 +151,56 @@ impl NativeMessage { } #[classmethod] - #[pyo3(signature = (json, *, ignore_unknown_fields = false, registry = None))] + #[pyo3(signature = (json, *, ignore_unknown_fields = false, registry = None, allocation_limit = None))] fn from_json<'py>( cls: &Bound<'py, PyType>, py: Python<'py>, json: &Bound<'py, PyAny>, ignore_unknown_fields: bool, registry: Option>, + allocation_limit: Option, ) -> PyResult> { let constants = Constants::get(py)?; let marshaler_any = cls.getattr(&constants.ext_marshaler)?; let marshaler = marshaler_any.cast::()?.get().clone(); - let message = marshaler.new_empty_message(py, cls)?; - parse_json_into( - py, - &marshaler, - &message, - json, - ignore_unknown_fields, - registry, - )?; - Ok(message) + with_budget!(allocation_limit, |budget| { + budget.charge_message(&marshaler)?; + let message = marshaler.new_empty_message(py, cls)?; + parse_json_into( + py, + &marshaler, + &message, + json, + ignore_unknown_fields, + registry, + budget, + )?; + Ok(message) + }) } - #[pyo3(signature = (json, *, ignore_unknown_fields = false, registry = None))] + #[pyo3(signature = (json, *, ignore_unknown_fields = false, registry = None, allocation_limit = None))] fn _merge_from_json( slf: &Bound<'_, Self>, py: Python<'_>, json: &Bound<'_, PyAny>, ignore_unknown_fields: bool, registry: Option>, + allocation_limit: Option, ) -> PyResult<()> { let marshaler = NativeMessage::get_marshaler(slf)?; - parse_json_into(py, &marshaler, slf, json, ignore_unknown_fields, registry) + with_budget!(allocation_limit, |budget| { + budget.charge_message(&marshaler)?; + parse_json_into( + py, + &marshaler, + slf, + json, + ignore_unknown_fields, + registry, + budget, + ) + }) } #[pyo3(signature = (*, registry = None, always_emit_implicit = false, print_enums_as_ints = false, use_proto_field_name = false))] @@ -203,24 +225,28 @@ impl NativeMessage { } #[classmethod] - #[pyo3(signature = (data, *, ignore_unknown_fields = false, registry = None))] + #[pyo3(signature = (data, *, ignore_unknown_fields = false, registry = None, allocation_limit = None))] fn _from_json_value<'py>( cls: &Bound<'py, PyType>, py: Python<'py>, data: Bound<'py, PyAny>, ignore_unknown_fields: bool, registry: Option>, + allocation_limit: Option, ) -> PyResult> { let constants = Constants::get(py)?; let marshaler_any = cls.getattr(&constants.ext_marshaler)?; let marshaler = marshaler_any.cast::()?.get().clone(); - let message = marshaler.new_empty_message(py, cls)?; - let opts = FromJsonOpts { - ignore_unknown_fields, - registry, - }; - read_message_from_tree(py, &marshaler, &message, data, &opts)?; - Ok(message) + with_budget!(allocation_limit, |budget| { + budget.charge_message(&marshaler)?; + let message = marshaler.new_empty_message(py, cls)?; + let opts = FromJsonOpts { + ignore_unknown_fields, + registry, + }; + read_message_from_tree(py, &marshaler, &message, data, &opts, budget)?; + Ok(message) + }) } fn __setattr__( @@ -264,14 +290,20 @@ impl NativeMessage { Ok(new) } + #[pyo3(signature = (data, ignore_unknown_fields, allocation_limit = None))] fn _merge_from_binary( slf: &Bound<'_, Self>, py: Python<'_>, data: Buffer, ignore_unknown_fields: bool, + allocation_limit: Option, ) -> PyResult<()> { let data = data.into_inner(); - NativeMessage::get_marshaler(slf)?.merge_from_binary(py, slf, data, ignore_unknown_fields) + let marshaler = NativeMessage::get_marshaler(slf)?; + with_budget!(allocation_limit, |budget| { + budget.charge_message(&marshaler)?; + marshaler.merge_from_binary(py, slf, data, ignore_unknown_fields, budget) + }) } fn __deepcopy__<'py>( @@ -514,18 +546,26 @@ fn parse_json_into<'py>( json: &Bound<'py, PyAny>, ignore_unknown_fields: bool, registry: Option>, + budget: &mut impl Budget, ) -> PyResult<()> { let opts = FromJsonOpts { ignore_unknown_fields, registry, }; if let Ok(text) = json.cast::() { - merge_from_json(py, marshaler, message, text.to_str()?.as_bytes(), &opts) + merge_from_json( + py, + marshaler, + message, + text.to_str()?.as_bytes(), + &opts, + budget, + ) } else if let Ok(bytes) = json.cast::() { - merge_from_json(py, marshaler, message, bytes.as_bytes(), &opts) + merge_from_json(py, marshaler, message, bytes.as_bytes(), &opts, budget) } else if let Ok(bytearray) = json.cast::() { let owned = bytearray.to_vec(); - merge_from_json(py, marshaler, message, &owned, &opts) + merge_from_json(py, marshaler, message, &owned, &opts, budget) } else { Err(PyTypeError::new_err(format!( "json must be str, bytes, or bytearray, got {}", diff --git a/packages/protobuf-py-ext/src/parser.rs b/packages/protobuf-py-ext/src/parser.rs index 717c6bc..b27ddec 100644 --- a/packages/protobuf-py-ext/src/parser.rs +++ b/packages/protobuf-py-ext/src/parser.rs @@ -23,6 +23,7 @@ use pyo3::{ use crate::{ attribute_access::AttributeAccess, + budget::Budget, constants::Constants, descriptor::{ DescEnum, DescField, DescFieldValue, DescMessage, DescSingleValue, FieldPresence, @@ -260,6 +261,7 @@ impl FieldParser { buffer: &mut Bytes, opts: FromBinaryOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { match &self.type_ { ParserFieldType::Singular { @@ -275,6 +277,7 @@ impl FieldParser { depth, oneof_attr.as_ref(), *requires_presence, + budget, )?, ParserFieldType::List { unpacked_wire_type, @@ -289,6 +292,7 @@ impl FieldParser { depth, *unpacked_wire_type, *packable, + budget, )?, ParserFieldType::Map { key_type, @@ -306,6 +310,7 @@ impl FieldParser { value_parser, key_default_value, value_default_value, + budget, )?, } @@ -323,6 +328,7 @@ impl FieldParser { depth: usize, oneof_attr: Option<&AttributeAccess>, requires_presence: bool, + budget: &mut impl Budget, ) -> PyResult<()> { let value = self.read_single_value( py, @@ -332,9 +338,13 @@ impl FieldParser { buffer, opts, depth, + budget, )?; match value { SingleValue::Parsed(value) => { + if oneof_attr.is_some() { + budget.charge_oneof()?; + } self.assign_singular(py, message, &value, oneof_attr, requires_presence)?; } SingleValue::UnknownEnumValue(number) => { @@ -342,7 +352,7 @@ impl FieldParser { let mut field = BytesMut::new(); encode_varint(tag as u64, &mut field); encode_varint(number as u64, &mut field); - write_unknown_field(py, message, tag >> 3, &field)?; + write_unknown_field(py, message, tag >> 3, &field, budget)?; } } } @@ -382,6 +392,7 @@ impl FieldParser { depth: usize, unpacked_wire_type: WireType, packable: bool, + budget: &mut impl Budget, ) -> PyResult<()> { let python_list = self.attr.get(py, message)?; let list = python_list.cast::()?; @@ -398,9 +409,13 @@ impl FieldParser { &mut list_buffer, opts, depth, + budget, )?; match value { - SingleValue::Parsed(value) => list.append(value)?, + SingleValue::Parsed(value) => { + budget.charge_list_item()?; + list.append(value)?; + } SingleValue::UnknownEnumValue(number) => { if !opts.ignore_unknown_fields { // Always add unknown values as unpacked. @@ -408,7 +423,7 @@ impl FieldParser { let tag = (tag & !0b111) | (unpacked_wire_type as u32); encode_varint(tag as u64, &mut field); encode_varint(number as u64, &mut field); - write_unknown_field(py, message, tag >> 3, &field)?; + write_unknown_field(py, message, tag >> 3, &field, budget)?; } } } @@ -422,15 +437,19 @@ impl FieldParser { buffer, opts, depth, + budget, )?; match value { - SingleValue::Parsed(value) => list.append(value)?, + SingleValue::Parsed(value) => { + budget.charge_list_item()?; + list.append(value)?; + } SingleValue::UnknownEnumValue(number) => { if !opts.ignore_unknown_fields { let mut field = BytesMut::new(); encode_varint(tag as u64, &mut field); encode_varint(number as u64, &mut field); - write_unknown_field(py, message, tag >> 3, &field)?; + write_unknown_field(py, message, tag >> 3, &field, budget)?; } } } @@ -450,6 +469,7 @@ impl FieldParser { value_parser: &FieldParser, key_default_value: &Py, value_default_value: &Py, + budget: &mut impl Budget, ) -> PyResult<()> { let len = decode_varint(buffer).map_err(map_varint_err)? as usize; check_buffer_remaining(buffer, len)?; @@ -471,9 +491,10 @@ impl FieldParser { entry_tag, &entry_checkpoint, opts, + budget, ); } - key = Some(read_scalar(py, key_type, &mut entry_buffer)?); + key = Some(read_scalar(py, key_type, &mut entry_buffer, budget)?); } 2 => { if value_parser.wire_type != wire_type { @@ -483,6 +504,7 @@ impl FieldParser { entry_tag, &entry_checkpoint, opts, + budget, ); } // A repeated value field within one entry merges into the @@ -499,6 +521,7 @@ impl FieldParser { &mut entry_buffer, opts, depth, + budget, )?); } _ => { @@ -512,14 +535,23 @@ impl FieldParser { value } else if let FieldParserValue::Message { message, .. } = &value_parser.value { // For message values, the default is a new instance of the message, not None. + budget.charge_message(message.get_marshaler(py)?)?; SingleValue::Parsed(message.get_python_type(py).call0()?) } else { SingleValue::Parsed(value_default_value.bind(py).clone()) }; let SingleValue::Parsed(value) = value else { // Unknown value (i.e., unknown closed enum value), whole entry is an unknown field. - return Self::read_unknown_map_entry(py, message, entry_tag, &entry_checkpoint, opts); + return Self::read_unknown_map_entry( + py, + message, + entry_tag, + &entry_checkpoint, + opts, + budget, + ); }; + budget.charge_dict_entry()?; self.assign_map_entry(py, message, key, value)?; Ok(()) } @@ -544,13 +576,14 @@ impl FieldParser { tag: u32, entry_bytes: &[u8], opts: FromBinaryOpts, + budget: &mut impl Budget, ) -> PyResult<()> { if !opts.ignore_unknown_fields { let mut field_bytes = BytesMut::new(); encode_varint(tag as u64, &mut field_bytes); encode_varint(entry_bytes.len() as u64, &mut field_bytes); field_bytes.extend_from_slice(entry_bytes); - write_unknown_field(py, message, tag >> 3, &field_bytes)?; + write_unknown_field(py, message, tag >> 3, &field_bytes, budget)?; } Ok(()) } @@ -565,9 +598,10 @@ impl FieldParser { buffer: &mut Bytes, opts: FromBinaryOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult> { let value = match &self.value { - FieldParserValue::Scalar(scalar_type) => read_scalar(py, *scalar_type, buffer)?, + FieldParserValue::Scalar(scalar_type) => read_scalar(py, *scalar_type, buffer, budget)?, FieldParserValue::Message { message: message_desc, .. @@ -595,6 +629,7 @@ impl FieldParser { { existing.cast_into::()? } else { + budget.charge_message(marshaler)?; marshaler.new_empty_message(py, parser.inner.python_type.bind(py))? }; parser.merge_from_binary( @@ -603,6 +638,7 @@ impl FieldParser { &mut message_buffer, opts, depth + 1, + budget, )?; message_instance.into_any() } @@ -612,6 +648,7 @@ impl FieldParser { if let Some(value) = value { value.bind(py).clone() } else if enum_.open { + budget.charge_enum()?; enum_.py_type.bind(py).call1((number,))? } else { return Ok(SingleValue::UnknownEnumValue(number)); @@ -709,6 +746,7 @@ impl MessageParser { buffer: &mut Bytes, opts: FromBinaryOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { check_parse_recursion_depth(depth)?; while buffer.has_remaining() { @@ -722,12 +760,18 @@ impl MessageParser { if let Some(field) = self.inner.fields.get(field_number) && field.wire_type_matches(wire_type) { - field.read_field(py, message, tag, wire_type, buffer, opts, depth)?; + field.read_field(py, message, tag, wire_type, buffer, opts, depth, budget)?; } else { skip_field_with_wire_type(field_number, wire_type, buffer, depth + 1)?; if !opts.ignore_unknown_fields { let field_len = checkpoint.len() - buffer.len(); - write_unknown_field(py, message, field_number, &checkpoint[..field_len])?; + write_unknown_field( + py, + message, + field_number, + &checkpoint[..field_len], + budget, + )?; } } } @@ -740,7 +784,26 @@ fn read_scalar<'py>( py: Python<'py>, s: ScalarType, buffer: &mut Bytes, + budget: &mut impl Budget, ) -> PyResult> { + match s { + ScalarType::Double | ScalarType::Float => budget.charge_float()?, + // Small ints are interned and larger ones vary a little in size, but + // a flat charge is close enough. + ScalarType::Int64 + | ScalarType::Uint64 + | ScalarType::Int32 + | ScalarType::Uint32 + | ScalarType::Fixed64 + | ScalarType::Fixed32 + | ScalarType::Sfixed32 + | ScalarType::Sfixed64 + | ScalarType::Sint32 + | ScalarType::Sint64 => budget.charge_int()?, + // Bools are shared singletons; strings and bytes are charged below + // once the length is known. + ScalarType::Bool | ScalarType::String | ScalarType::Bytes => {} + } let res = match s { ScalarType::Double => { PyFloat::new(py, buffer.try_get_f64_le().map_err(map_try_get_err)?).into_any() @@ -774,12 +837,14 @@ fn read_scalar<'py>( ScalarType::String => { let len = decode_varint(buffer).map_err(map_varint_err)? as usize; check_buffer_remaining(buffer, len)?; + budget.charge_str(&buffer[..len])?; let bytes = buffer.split_to(len); PyString::from_bytes(py, &bytes)?.into_any() } ScalarType::Bytes => { let len = decode_varint(buffer).map_err(map_varint_err)? as usize; check_buffer_remaining(buffer, len)?; + budget.charge_bytes(len)?; let bytes = buffer.split_to(len); PyBytes::new(py, &bytes).into_any() } @@ -903,16 +968,29 @@ fn write_unknown_field( message: &Bound<'_, NativeMessage>, field_number: u32, field_bytes: &[u8], + budget: &mut impl Budget, ) -> PyResult<()> { + if message.get().unknown_fields(py).is_none() { + budget.charge_dict()?; + } let unknown_fields_unbound = message.get().get_or_init_unknown_fields(py); let unknown_fields = unknown_fields_unbound.bind(py); let field_list = if let Ok(list) = unknown_fields.get_item(field_number) { list.cast::()?.clone() } else { + // The list and the dict entry holding it, plus the key unless it is + // one of the small ints that are shared singletons. + if field_number > 256 { + budget.charge_int()?; + } + budget.charge_list()?; + budget.charge_dict_entry()?; let new_list = PyList::empty(py); unknown_fields.set_item(field_number, &new_list)?; new_list }; + budget.charge_bytes(field_bytes.len())?; + budget.charge_list_item()?; field_list.append(PyBytes::new(py, field_bytes))?; Ok(()) } diff --git a/packages/protobuf-py-ext/src/wkt_registry.rs b/packages/protobuf-py-ext/src/wkt_registry.rs index 69811f7..fa18d3a 100644 --- a/packages/protobuf-py-ext/src/wkt_registry.rs +++ b/packages/protobuf-py-ext/src/wkt_registry.rs @@ -9,13 +9,14 @@ use pyo3::{ Bound, IntoPyObjectExt as _, Py, PyAny, PyResult, Python, exceptions::{PyTypeError, PyValueError}, types::{ - PyAnyMethods as _, PyBool, PyDict, PyDictMethods as _, PyFloat, PyInt, PyList, - PyListMethods as _, PyString, PyStringMethods as _, PyType, + PyAnyMethods as _, PyBool, PyBytesMethods as _, PyDict, PyDictMethods as _, PyFloat, PyInt, + PyList, PyListMethods as _, PyString, PyStringMethods as _, PyType, }, }; use crate::{ attribute_access::AttributeAccess, + budget::{Budget, NoopBudget}, constants::Constants, descriptor::{DescField, DescFieldValue, DescMessage, DescSingleValue, ScalarType}, json_parse::{FieldContext, FromJsonOpts, read_json_value, read_message, read_scalar}, @@ -58,11 +59,14 @@ impl WktTimestamp { src: &mut R, _opts: &FromJsonOpts, _depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); - let (secs, nanos) = with_wkt_str(marshaler, src, |text| { + let (secs, nanos) = with_wkt_str(marshaler, src, budget, |text, _| { parse_timestamp(&marshaler.type_name, text) })?; + budget.charge_int()?; + budget.charge_int()?; self.seconds .set(message.as_any(), PyInt::new(py, secs).as_any())?; self.nanos @@ -99,11 +103,14 @@ impl WktDuration { src: &mut R, _opts: &FromJsonOpts, _depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); - let (secs, nanos) = with_wkt_str(marshaler, src, |text| { + let (secs, nanos) = with_wkt_str(marshaler, src, budget, |text, _| { parse_duration(&marshaler.type_name, text) })?; + budget.charge_int()?; + budget.charge_int()?; self.seconds .set(message.as_any(), PyInt::new(py, secs).as_any())?; self.nanos @@ -157,7 +164,9 @@ impl WktAny { let value = self.value.get(py, message.as_any())?.extract::()?; let inner_msg = inner_marshaler.new_empty_message(py, &inner_type)?; - inner_marshaler.merge_from_binary(py, &inner_msg, value, false)?; + // While we go through parsing to serialize a any to JSON, we don't need to track budget since this is + // not a real parse operation. + inner_marshaler.merge_from_binary(py, &inner_msg, value, false, &mut NoopBudget)?; sink.begin_object()?; if inner_marshaler.wkt.is_none() { @@ -183,12 +192,13 @@ impl WktAny { src: &mut R, opts: &FromJsonOpts, _depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); let message_type_name = &marshaler.type_name; // For a string input, this will eagerly parse out a whole dictionary since // we need to first find `@type` before parsing. - let tree = read_json_value(src)?; + let tree = read_json_value(src, budget)?; let Ok(dict) = tree.cast::() else { return Err(PyTypeError::new_err(format!( "cannot decode {message_type_name} from JSON: {}", @@ -230,6 +240,7 @@ impl WktAny { .getattr(&marshaler.constants.ext_marshaler)? .cast_into::()?; let inner_marshaler = inner_marshaler.get(); + budget.charge_message(inner_marshaler)?; let inner_msg = inner_marshaler.new_empty_message(py, &inner_type)?; let is_wkt = inner_marshaler.wkt.is_some(); @@ -238,17 +249,19 @@ impl WktAny { .get_item("value")? .unwrap_or_else(|| py.None().into_bound(py)); let mut sub = PyTreeSource::new(py, value); - read_message(inner_marshaler, &inner_msg, &mut sub, opts, 1)?; + read_message(inner_marshaler, &inner_msg, &mut sub, opts, 1, budget)?; } else { let copy = dict.copy()?; copy.del_item("@type")?; let mut sub = PyTreeSource::new(py, copy.into_any()); - read_message(inner_marshaler, &inner_msg, &mut sub, opts, 1)?; + read_message(inner_marshaler, &inner_msg, &mut sub, opts, 1, budget)?; } // Any.pack let packed_url = format!("type.googleapis.com/{}", inner_marshaler.type_name); let packed_value = inner_marshaler.to_binary(py, &inner_msg, true)?; + budget.charge_str(packed_url.as_bytes())?; + budget.charge_bytes(packed_value.as_bytes().len())?; self.type_url .set(message.as_any(), &PyString::new(py, &packed_url).into_any())?; self.value.set(message.as_any(), packed_value.as_any())?; @@ -300,13 +313,14 @@ impl WktFieldMask { src: &mut R, _opts: &FromJsonOpts, _depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); let paths = self .paths .get(py, message.as_any())? .cast_into::()?; - with_wkt_str(marshaler, src, |text| { + with_wkt_str(marshaler, src, budget, |text, budget| { if text.is_empty() { return Ok(()); } @@ -317,7 +331,10 @@ impl WktFieldMask { marshaler.type_name ))); } - paths.append(buffa_wkt::camel_to_snake(part))?; + let path = buffa_wkt::camel_to_snake(part); + budget.charge_str(path.as_bytes())?; + budget.charge_list_item()?; + paths.append(path)?; } Ok(()) }) @@ -360,10 +377,11 @@ impl WktStruct { src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); if src.peek()? != JsonKind::Object { - let json = read_json_value(src)?; + let json = read_json_value(src, budget)?; return Err(PyTypeError::new_err(format!( "cannot decode {} from JSON: {}", marshaler.type_name, @@ -377,9 +395,12 @@ impl WktStruct { .cast_into::()?; // Duplicate keys use last-in-wins semantics (per the ProtoJSON spec). src.for_each_object_key(|key, src| { + budget.charge_message(value_marshaler)?; let value_msg = value_marshaler.new_empty_message(py, self.value.get_python_type(py))?; - read_message(value_marshaler, &value_msg, src, opts, depth + 1)?; + read_message(value_marshaler, &value_msg, src, opts, depth + 1, budget)?; + budget.charge_str(key.as_bytes())?; + budget.charge_dict_entry()?; dict.set_item(key, value_msg)?; Ok(()) })?; @@ -422,10 +443,11 @@ impl WktListValue { src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); if src.peek()? != JsonKind::Array { - let json = read_json_value(src)?; + let json = read_json_value(src, budget)?; return Err(PyTypeError::new_err(format!( "cannot decode {} from JSON: {}", marshaler.type_name, @@ -438,9 +460,11 @@ impl WktListValue { .get(py, message.as_any())? .cast_into::()?; src.for_each_array_item(|src| { + budget.charge_message(element_marshaler)?; let value_msg = element_marshaler.new_empty_message(py, self.element.get_python_type(py))?; - read_message(element_marshaler, &value_msg, src, opts, depth + 1)?; + read_message(element_marshaler, &value_msg, src, opts, depth + 1, budget)?; + budget.charge_list_item()?; list.append(value_msg)?; Ok(()) })?; @@ -510,8 +534,10 @@ impl WktValue { src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); + budget.charge_oneof()?; let oneof = match src.peek()? { JsonKind::Null => { src.next_null()?; @@ -523,27 +549,30 @@ impl WktValue { ), JsonKind::Number => { let number = src.next_float()?; + budget.charge_float()?; Oneof::new( self.number_name.bind(py), &PyFloat::new(py, number).into_any(), ) } JsonKind::String => { - let string = src.next_py_str()?; + let string = src.next_py_str(budget)?; Oneof::new(self.string_name.bind(py), &string) } JsonKind::Array => { let desc = &self.list_message; let inner = desc.get_marshaler(py)?; + budget.charge_message(inner)?; let list_msg = inner.new_empty_message(py, desc.get_python_type(py))?; - read_message(inner, &list_msg, src, opts, depth + 1)?; + read_message(inner, &list_msg, src, opts, depth + 1, budget)?; Oneof::new(self.list_name.bind(py), &list_msg.into_any()) } JsonKind::Object => { let desc = &self.struct_message; let inner = desc.get_marshaler(py)?; + budget.charge_message(inner)?; let struct_msg = inner.new_empty_message(py, desc.get_python_type(py))?; - read_message(inner, &struct_msg, src, opts, depth + 1)?; + read_message(inner, &struct_msg, src, opts, depth + 1, budget)?; Oneof::new(self.struct_name.bind(py), &struct_msg.into_any()) } }; @@ -581,6 +610,7 @@ impl WktWrapper { src: &mut R, _opts: &FromJsonOpts, _depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { let py = src.py(); if src.peek()? == JsonKind::Null { @@ -591,7 +621,7 @@ impl WktWrapper { } let name = self.name.bind(py); let ctx = FieldContext::Field { marshaler, name }; - let value = read_scalar(&ctx, src, self.scalar)?; + let value = read_scalar(&ctx, src, self.scalar, budget)?; self.field.set(message.as_any(), &value) } } @@ -639,16 +669,17 @@ impl WktKind { src: &mut R, opts: &FromJsonOpts, depth: usize, + budget: &mut impl Budget, ) -> PyResult<()> { match self { - WktKind::Timestamp(w) => w.read_json(marshaler, message, src, opts, depth), - WktKind::Duration(w) => w.read_json(marshaler, message, src, opts, depth), - WktKind::Any(w) => w.read_json(marshaler, message, src, opts, depth), - WktKind::FieldMask(w) => w.read_json(marshaler, message, src, opts, depth), - WktKind::Struct(w) => w.read_json(marshaler, message, src, opts, depth), - WktKind::ListValue(w) => w.read_json(marshaler, message, src, opts, depth), - WktKind::Value(w) => w.read_json(marshaler, message, src, opts, depth), - WktKind::Wrapper(w) => w.read_json(marshaler, message, src, opts, depth), + WktKind::Timestamp(w) => w.read_json(marshaler, message, src, opts, depth, budget), + WktKind::Duration(w) => w.read_json(marshaler, message, src, opts, depth, budget), + WktKind::Any(w) => w.read_json(marshaler, message, src, opts, depth, budget), + WktKind::FieldMask(w) => w.read_json(marshaler, message, src, opts, depth, budget), + WktKind::Struct(w) => w.read_json(marshaler, message, src, opts, depth, budget), + WktKind::ListValue(w) => w.read_json(marshaler, message, src, opts, depth, budget), + WktKind::Value(w) => w.read_json(marshaler, message, src, opts, depth, budget), + WktKind::Wrapper(w) => w.read_json(marshaler, message, src, opts, depth, budget), } } @@ -884,20 +915,21 @@ fn match_wrapper(fields: &[DescField], by_name: &HashMap) -> Opti } } -fn with_wkt_str<'py, S: JsonSource<'py>, R>( +fn with_wkt_str<'py, S: JsonSource<'py>, B: Budget, R>( marshaler: &MessageMarshaler, src: &mut S, - f: impl FnOnce(&str) -> PyResult, + budget: &mut B, + f: impl FnOnce(&str, &mut B) -> PyResult, ) -> PyResult { if src.peek()? != JsonKind::String { - let value = read_json_value(src)?; + let value = read_json_value(src, budget)?; return Err(PyTypeError::new_err(format!( "cannot decode {} from JSON: {}", marshaler.type_name, value.str()? ))); } - src.with_next_str(f) + src.with_next_str(|text| f(text, budget)) } fn type_url_to_name(url: &str) -> PyResult<&str> { diff --git a/src/protobuf/_budget.py b/src/protobuf/_budget.py new file mode 100644 index 0000000..81277a4 --- /dev/null +++ b/src/protobuf/_budget.py @@ -0,0 +1,337 @@ +# Copyright (c) 2025-2026 Buf Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Protocol, cast + +from ._descriptors import DescMessage, ScalarType + +if TYPE_CHECKING: + from collections.abc import Sized + +# Approximate sizes of CPython heap allocations, measured with `sys.getsizeof` +# on 64-bit CPython 3.14. The budget guards against unbounded allocation from +# malicious payloads rather than providing exact accounting, so small +# inaccuracies across versions and builds are fine. + +OBJECT_HEADER_SIZE = 16 +"""Refcount and type pointer at the start of every object.""" +SLOT_SIZE = 8 +"""One `__slots__` entry: a pointer.""" +GC_HEAD_SIZE = 16 +"""GC header allocated in front of every GC-tracked object.""" +FLOAT_SIZE = 24 +"""A float object.""" +INT_SIZE = 36 +"""A 64-bit int object. Smaller ints are slightly smaller.""" +STR_OVERHEAD = 41 +"""Header of an ASCII str; the length is added on top.""" +BYTES_OVERHEAD = 33 +"""Header of a bytes object.""" +EMPTY_LIST_SIZE = 56 +"""An empty list.""" +EMPTY_DICT_SIZE = 64 +"""An empty dict.""" +LIST_SLOT_SIZE = 8 +"""One appended list element: an 8-byte pointer slot.""" +DICT_ENTRY_SIZE = 40 +"""One inserted dict entry: hash + key + value words plus growth slack.""" +ONEOF_SIZE = 32 +"""A Oneof wrapper object: object header plus two references.""" + +# A str stores every character at the width of its widest one, with a header +# (including the trailing NUL) that depends on that width. +_STR_LATIN1_OVERHEAD = 57 +_STR_UCS2_OVERHEAD = 58 +_STR_UCS4_OVERHEAD = 60 +_UCS2_OR_WIDER = re.compile("[\u0100-\U0010ffff]") +_UCS4 = re.compile("[\U00010000-\U0010ffff]") + + +def str_size(value: str) -> int: + """The size of a str.""" + length = len(value) + if value.isascii(): + return STR_OVERHEAD + length + if _UCS4.search(value): + return _STR_UCS4_OVERHEAD + 4 * length + if _UCS2_OR_WIDER.search(value): + return _STR_UCS2_OVERHEAD + 2 * length + return _STR_LATIN1_OVERHEAD + length + + +def _instance_size(message_type: type) -> int: + """The fixed allocation size of an instance of a slots-only class.""" + basicsize = getattr(message_type, "__basicsize__", None) + if basicsize is not None: + return basicsize + # PyPy doesn't provide basicsize so we approximate it based on slot count. + slots = 0 + for cls in message_type.__mro__: + names = cls.__dict__.get("__slots__", ()) + names = (names,) if isinstance(names, str) else names + slots += sum(1 for name in names if name not in ("__dict__", "__weakref__")) + return OBJECT_HEADER_SIZE + SLOT_SIZE * slots + + +def _message_alloc_size(desc: DescMessage) -> int: + """Approximate heap size of a freshly-initialized message instance. + + The fixed instance size (all fields are slots) plus the empty containers + created for repeated/map field defaults. + """ + size = _instance_size(desc.type) + GC_HEAD_SIZE + for _, default in desc._defaults: + if isinstance(default, list): + size += EMPTY_LIST_SIZE + elif isinstance(default, dict): + size += EMPTY_DICT_SIZE + return size + + +# Allocation for each scalar type whose values have a fixed size, indexed by +# the type's number. Bools are shared singletons and allocate nothing. +_FIXED_SCALAR_SIZES = ( + None, # 0: unused + FLOAT_SIZE, # 1: DOUBLE + FLOAT_SIZE, # 2: FLOAT + INT_SIZE, # 3: INT64 + INT_SIZE, # 4: UINT64 + INT_SIZE, # 5: INT32 + INT_SIZE, # 6: FIXED64 + INT_SIZE, # 7: FIXED32 + 0, # 8: BOOL + None, # 9: STRING + None, # 10: GROUP + None, # 11: MESSAGE + None, # 12: BYTES + INT_SIZE, # 13: UINT32 + None, # 14: ENUM + INT_SIZE, # 15: SFIXED32 + INT_SIZE, # 16: SFIXED64 + INT_SIZE, # 17: SINT32 + INT_SIZE, # 18: SINT64 +) + + +class Budget(Protocol): + """Accounts for the allocations made while parsing a message.""" + + def charge_message(self, desc: DescMessage) -> None: + """Charges for a new instance of the message type.""" + ... + + def charge_scalar(self, scalar_type: ScalarType, value: object) -> None: + """Charges for a parsed scalar value of the given type.""" + ... + + def charge_fixed_scalar(self, scalar_type: ScalarType) -> None: + """Charges for a scalar of a type whose values have a fixed size. + + These are all scalar types except strings and bytes. + """ + ... + + def charge_str(self, value: str) -> None: + """Charges for a str.""" + ... + + def charge_str_encoded(self, length: int) -> None: + """Charges the least a str decoded from UTF-8 of the given length can be. + + A str is stored at the width of its widest character, which is only + known once decoded. The least is where every character takes two bytes + of UTF-8 and one in the str. + """ + ... + + def charge_str_decoded(self, value: str, length: int) -> None: + """Charges the rest of a str after charge_str_encoded for its UTF-8 length.""" + ... + + def charge_bytes(self, length: int) -> None: + """Charges for a bytes of the given length.""" + ... + + def charge_int(self) -> None: + """Charges for an int. + + Small ints are shared and larger ones vary a little in size, but a + flat charge is close enough. + """ + ... + + def charge_float(self) -> None: + """Charges for a float.""" + ... + + def charge_enum(self) -> None: + """Charges for a member of an enum type created for a number it does not define.""" + ... + + def charge_oneof(self) -> None: + """Charges for a Oneof wrapper.""" + ... + + def charge_list(self) -> None: + """Charges for a new list.""" + ... + + def charge_list_item(self) -> None: + """Charges for appending an item to a list.""" + ... + + def charge_dict(self) -> None: + """Charges for a new dict.""" + ... + + def charge_dict_entry(self) -> None: + """Charges for inserting an entry into a dict.""" + ... + + +class TrackingBudget: + """Tracks approximate allocations, raising an error once a limit is exceeded.""" + + __slots__ = ("current", "max") + + def __init__(self, limit: int) -> None: + self.current = 0 + self.max = limit + + def charge(self, amount: int) -> None: + """Charges the given number of bytes.""" + self.current += amount + if self.current > self.max: + msg = f"allocation budget exceeded: needed {self.current} bytes, limit is {self.max}" + raise ValueError(msg) + + def charge_message(self, desc: DescMessage) -> None: + size = desc._alloc_size + if size is None: + size = _message_alloc_size(desc) + object.__setattr__(desc, "_alloc_size", size) + self.charge(size) + + def charge_scalar(self, scalar_type: ScalarType, value: object) -> None: + if scalar_type == ScalarType.STRING: + self.charge(str_size(cast("str", value))) + elif scalar_type == ScalarType.BYTES: + self.charge(BYTES_OVERHEAD + len(cast("Sized", value))) + elif scalar_type == ScalarType.BOOL: + # Bools are shared singletons; nothing is allocated. + pass + elif scalar_type in (ScalarType.DOUBLE, ScalarType.FLOAT): + self.charge(FLOAT_SIZE) + else: + self.charge(INT_SIZE) + + def charge_fixed_scalar(self, scalar_type: ScalarType) -> None: + size = _FIXED_SCALAR_SIZES[scalar_type.value] + assert size is not None # noqa: S101 + self.charge(size) + + def charge_str(self, value: str) -> None: + self.charge(str_size(value)) + + def charge_str_encoded(self, length: int) -> None: + self.charge(STR_OVERHEAD + length // 2) + + def charge_str_decoded(self, value: str, length: int) -> None: + self.charge(str_size(value) - (STR_OVERHEAD + length // 2)) + + def charge_bytes(self, length: int) -> None: + self.charge(BYTES_OVERHEAD + length) + + def charge_int(self) -> None: + self.charge(INT_SIZE) + + def charge_float(self) -> None: + self.charge(FLOAT_SIZE) + + def charge_enum(self) -> None: + self.charge(INT_SIZE + GC_HEAD_SIZE) + + def charge_oneof(self) -> None: + self.charge(ONEOF_SIZE) + + def charge_list(self) -> None: + self.charge(EMPTY_LIST_SIZE) + + def charge_list_item(self) -> None: + self.charge(LIST_SLOT_SIZE) + + def charge_dict(self) -> None: + self.charge(EMPTY_DICT_SIZE) + + def charge_dict_entry(self) -> None: + self.charge(DICT_ENTRY_SIZE) + + +class NoopBudget: + """Does no accounting, for parsing without an allocation limit.""" + + __slots__ = () + + def charge_message(self, desc: DescMessage) -> None: + pass + + def charge_scalar(self, scalar_type: ScalarType, value: object) -> None: + pass + + def charge_fixed_scalar(self, scalar_type: ScalarType) -> None: + pass + + def charge_str(self, value: str) -> None: + pass + + def charge_str_encoded(self, length: int) -> None: + pass + + def charge_str_decoded(self, value: str, length: int) -> None: + pass + + def charge_bytes(self, length: int) -> None: + pass + + def charge_int(self) -> None: + pass + + def charge_float(self) -> None: + pass + + def charge_enum(self) -> None: + pass + + def charge_oneof(self) -> None: + pass + + def charge_list(self) -> None: + pass + + def charge_list_item(self) -> None: + pass + + def charge_dict(self) -> None: + pass + + def charge_dict_entry(self) -> None: + pass + + +def new_budget(limit: int | None) -> Budget: + """Returns the budget for an optional allocation limit.""" + return NoopBudget() if limit is None else TrackingBudget(limit) diff --git a/src/protobuf/_descriptors.py b/src/protobuf/_descriptors.py index f087e7e..4d9946c 100644 --- a/src/protobuf/_descriptors.py +++ b/src/protobuf/_descriptors.py @@ -310,6 +310,10 @@ class DescMessage: repr=False, compare=False, hash=False, init=False ) """Whether this message has any fields that require separate presence tracking.""" + _alloc_size: int | None = dataclassfield( + default=None, repr=False, compare=False, hash=False, init=False + ) + """Approximate heap size of a new instance, for the parsing allocation budget.""" def _finish_init(self) -> None: """Finish initialization of private attributes. diff --git a/src/protobuf/_from_binary.py b/src/protobuf/_from_binary.py index 49e99c7..728f5f5 100644 --- a/src/protobuf/_from_binary.py +++ b/src/protobuf/_from_binary.py @@ -14,11 +14,12 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, TypeVar from typing_extensions import Buffer, assert_never +from ._budget import Budget, NoopBudget from ._descriptors import ( DescEnum, DescFieldValueEnum, @@ -47,9 +48,12 @@ class FromBinaryOptions: Args: ignore_unknown_fields: If `True`, unknown fields are ignored instead of being added to the message. + budget: Tracks approximate allocations during the parse and raises + once the configured limit is exceeded. Unlimited by default. """ ignore_unknown_fields: bool = False + budget: Budget = field(default_factory=NoopBudget) # Dispatch table for reading scalar values. CPython currently does not generate @@ -66,10 +70,10 @@ class FromBinaryOptions: BinaryReader.fixed64, # 6: FIXED64 BinaryReader.fixed32, # 7: FIXED32 BinaryReader.bool_, # 8: BOOL - BinaryReader.string, # 9: STRING + None, # 9: STRING (length-delimited, handled in read_scalar) None, # 10: GROUP None, # 11: MESSAGE - BinaryReader.bytes_, # 12: BYTES + None, # 12: BYTES (length-delimited, handled in read_scalar) BinaryReader.uint32, # 13: UINT32 None, # 14: ENUM BinaryReader.sfixed32, # 15: SFIXED32 @@ -79,9 +83,22 @@ class FromBinaryOptions: ) -def read_scalar(scalar_type: ScalarType, reader: BinaryReader) -> Any: +def read_scalar(scalar_type: ScalarType, reader: BinaryReader, budget: Budget) -> Any: + if scalar_type == ScalarType.STRING: + length = reader.varint() + # The size of a str is only known once decoded, so the least it can + # be is charged before and the rest after. + budget.charge_str_encoded(length) + value = str(reader.read(length), "utf-8") + budget.charge_str_decoded(value, length) + return value + if scalar_type == ScalarType.BYTES: + length = reader.varint() + budget.charge_bytes(length) + return bytes(reader.read(length)) reader_method = _SCALAR_READERS[scalar_type.value] assert reader_method is not None # noqa: S101 + budget.charge_fixed_scalar(scalar_type) return reader_method(reader) @@ -98,6 +115,32 @@ def _encode_varint(value: int) -> bytes: return bytes(result) +def _append_unknown_field( + message: Message, + field_number: int, + key_raw: bytes, + value_raw: bytes | memoryview, + budget: Budget, +) -> None: + """Records a raw field in the message's unknown fields, charging for it.""" + unknown_fields = message._unknown_fields + if unknown_fields is None: + budget.charge_dict() + unknown_fields = message._get_or_init_unknown_fields() + fields = unknown_fields.get(field_number) + if fields is None: + # The list and the dict entry holding it, plus the key unless it is + # one of the small ints that are shared singletons. + if field_number > 256: + budget.charge_int() + budget.charge_list() + budget.charge_dict_entry() + fields = unknown_fields[field_number] = [] + budget.charge_bytes(len(key_raw) + len(value_raw)) + budget.charge_list_item() + fields.append(key_raw + bytes(value_raw)) + + def read_message( message: Message, reader: BinaryReader, @@ -128,19 +171,26 @@ def read_message( field_raw = reader.skip(tag.wire_type, depth + 1, field_number=tag.number) if not opts.ignore_unknown_fields: key_raw = _encode_varint((tag.number << 3) | tag.wire_type) - message._get_or_init_unknown_fields().setdefault(tag.number, []).append( - key_raw + bytes(field_raw) + _append_unknown_field( + message, tag.number, key_raw, field_raw, opts.budget ) continue + budget = opts.budget match field_value := desc_field.value: case DescFieldValueScalar(): - message._set_member(desc_field, read_scalar(field_value.scalar, reader)) + value = read_scalar(field_value.scalar, reader, budget) + if field_value.oneof is not None: + budget.charge_oneof() + message._set_member(desc_field, value) case DescFieldValueMessage( message=desc_nested_message, delimited_encoding=delimited_encoding ): existing: Message | None = message._get_member(desc_field) if existing is None: + budget.charge_message(desc_nested_message) + if field_value.oneof is not None: + budget.charge_oneof() existing = desc_nested_message.type() message._set_member(desc_field, existing) if delimited_encoding: @@ -152,11 +202,13 @@ def read_message( existing, reader, opts, depth + 1, length=reader.varint() ) case DescFieldValueEnum(): - value = read_enum(field_value.enum, reader) + value = read_enum(field_value.enum, reader, budget) if isinstance(value, Enum): + if field_value.oneof is not None: + budget.charge_oneof() message._set_member(desc_field, value) elif not opts.ignore_unknown_fields: - _write_unknown_enum_field(message, desc_field.number, value) + _write_unknown_enum_field(message, desc_field.number, value, budget) case DescFieldValueList(): read_list( message, @@ -174,6 +226,7 @@ def read_message( ) if entry: key, value = entry + budget.charge_dict_entry() message._get_member(desc_field)[key] = value case _: assert_never(desc_field) @@ -204,15 +257,17 @@ def read_list( field_bytes = reader.skip(wire_type, depth + 1, field_number=field_number) if not opts.ignore_unknown_fields and message: key_raw = _encode_varint((field_number << 3) | wire_type) - message._get_or_init_unknown_fields().setdefault(field_number, []).append( - key_raw + bytes(field_bytes) + _append_unknown_field( + message, field_number, key_raw, field_bytes, opts.budget ) return + budget = opts.budget match element_type: case ScalarType(): - value = read_scalar(element_type, reader) + value = read_scalar(element_type, reader, budget) case DescMessage(): + budget.charge_message(element_type) if field_value.delimited_encoding: value = read_message( element_type.type(), @@ -226,13 +281,14 @@ def read_list( element_type.type(), reader, opts, depth + 1, length=reader.varint() ) case DescEnum(): - value = read_enum(element_type, reader) + value = read_enum(element_type, reader, budget) if not isinstance(value, Enum): if not opts.ignore_unknown_fields: - _write_unknown_enum_field(message, field_number, value) + _write_unknown_enum_field(message, field_number, value, budget) return case _: assert_never(element_type) + budget.charge_list_item() list_.append(value) @@ -246,39 +302,43 @@ def _read_packed_list( ) -> None: length = reader.varint() end = reader.offset + length + budget = opts.budget while reader.offset < end: match element_type: case ScalarType(): - list_.append(read_scalar(element_type, reader)) + value = read_scalar(element_type, reader, budget) + budget.charge_list_item() + list_.append(value) case DescEnum(): - value = read_enum(element_type, reader) + value = read_enum(element_type, reader, budget) if isinstance(value, Enum): + budget.charge_list_item() list_.append(value) elif not opts.ignore_unknown_fields: # Even for packed fields we write unknown enum values as unpacked. - _write_unknown_enum_field(message, field_number, value) + _write_unknown_enum_field(message, field_number, value, budget) case _: assert_never(element_type) -def read_enum(desc_enum: DescEnum, reader: BinaryReader) -> Enum | int: +def read_enum(desc_enum: DescEnum, reader: BinaryReader, budget: Budget) -> Enum | int: value = reader.int32() - if not desc_enum.open and not desc_enum._values_by_number.get(value): - return value + if not desc_enum._values_by_number.get(value): + if not desc_enum.open: + return value + budget.charge_enum() return desc_enum.type(value) def _write_unknown_enum_field( - message: Message | None, field_number: int, value: int + message: Message | None, field_number: int, value: int, budget: Budget ) -> None: if message is None: return writer = BinaryWriter() writer.tag(field_number, WireType.VARINT) writer.int32(value) - message._get_or_init_unknown_fields().setdefault(field_number, []).append( - writer.finish() - ) + _append_unknown_field(message, field_number, writer.finish(), b"", budget) def read_map_entry( @@ -297,6 +357,7 @@ def read_map_entry( value: Any = None message_value: Message | None = None + budget = opts.budget while reader.offset < end: tag = reader.tag() if tag.number == 1: # key @@ -305,7 +366,7 @@ def read_map_entry( message, field_number, reader, start_offset, opts, depth ) return None - key = read_scalar(field_value.key, reader) + key = read_scalar(field_value.key, reader, budget) elif tag.number == 2: # value if tag.wire_type != field_value._value_wire_type: _read_unknown_map_entry( @@ -314,9 +375,9 @@ def read_map_entry( return None match field_value.value: case ScalarType() as scalar_type: - value = read_scalar(scalar_type, reader) + value = read_scalar(scalar_type, reader, budget) case DescEnum() as desc_enum: - value = read_enum(desc_enum, reader) + value = read_enum(desc_enum, reader, budget) if not isinstance(value, Enum): _read_unknown_map_entry( message, field_number, reader, start_offset, opts, depth @@ -326,6 +387,7 @@ def read_map_entry( # A repeated value field within one entry merges into the # previous value, like any other singular message field. if message_value is None: + budget.charge_message(field_value.value) message_value = field_value.value.type() read_message( message_value, reader, opts, depth + 1, length=reader.varint() @@ -345,6 +407,7 @@ def read_map_entry( case DescEnum() as desc_enum: value = desc_enum.type(desc_enum.values[0].number) case DescMessage(): + budget.charge_message(field_value.value) value = field_value.value.type() case _: assert_never(field_value.value) @@ -368,13 +431,15 @@ def _read_unknown_map_entry( ) if not opts.ignore_unknown_fields and message: key_raw = _encode_varint((field_number << 3) | WireType.LENGTH_DELIMITED) - message._get_or_init_unknown_fields().setdefault(field_number, []).append( - key_raw + bytes(entry_bytes) - ) + _append_unknown_field(message, field_number, key_raw, entry_bytes, opts.budget) def merge_from_binary( - message: Message, data: Buffer, *, ignore_unknown_fields: bool = False + message: Message, + data: Buffer, + *, + ignore_unknown_fields: bool = False, + allocation_limit: int | None = None, ) -> None: """Parse serialized binary data, merging fields into an existing message. @@ -390,5 +455,9 @@ def merge_from_binary( message: The message instance to merge into. data: Serialized binary protobuf data. Must not be mutated during parsing. ignore_unknown_fields: If `True`, unknown fields in the binary data are silently discarded. + allocation_limit: If set, the approximate number of bytes of Python + objects the parse may allocate before raising a ValueError. """ - message._merge_from_binary(data, ignore_unknown_fields) + message._merge_from_binary( + data, ignore_unknown_fields, allocation_limit=allocation_limit + ) diff --git a/src/protobuf/_from_json.py b/src/protobuf/_from_json.py index 0aca051..d86f5c5 100644 --- a/src/protobuf/_from_json.py +++ b/src/protobuf/_from_json.py @@ -16,11 +16,12 @@ import math from base64 import b64decode -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Literal, TypeVar, cast from typing_extensions import assert_never +from ._budget import Budget, NoopBudget from ._descriptors import ( DescEnum, DescExtension, @@ -70,6 +71,8 @@ class FromJsonOptions: registry: Registry | None depth: int = 0 """Current message nesting depth. Mutated during a parse.""" + budget: Budget = field(default_factory=NoopBudget) + """Allocation budget shared across the whole parse. Unlimited by default.""" def _enter_message(opts: FromJsonOptions) -> FromJsonOptions: @@ -86,6 +89,7 @@ def merge_from_json( *, ignore_unknown_fields: bool = False, registry: Registry | None = None, + allocation_limit: int | None = None, ) -> None: """Parse a ProtoJSON string, merging fields into an existing message. @@ -103,6 +107,8 @@ def merge_from_json( silently discarded instead of raising an error. registry: Required to read google.protobuf.Any and extensions from JSON format. + allocation_limit: If set, the approximate number of bytes of Python + objects the parse may allocate before raising a ValueError. Raises: json.JSONDecodeError: If json_source is not valid JSON. @@ -111,7 +117,10 @@ def merge_from_json( through the registry. """ message._merge_from_json( - json, ignore_unknown_fields=ignore_unknown_fields, registry=registry + json, + ignore_unknown_fields=ignore_unknown_fields, + registry=registry, + allocation_limit=allocation_limit, ) @@ -179,7 +188,7 @@ def _read_field( ) -> None: match field_value := field.value: case DescFieldValueScalar(): - _read_scalar_field(msg, field, field_value, json) + _read_scalar_field(msg, field, field_value, json, opts.budget) case DescFieldValueEnum(): _read_enum_field(msg, field, field_value, json, opts) case DescFieldValueMessage(): @@ -197,7 +206,7 @@ def _read_extension( ) -> None: match field_value := ext.value: case DescFieldValueScalar(): - _read_scalar_extension(msg, ext, field_value, json) + _read_scalar_extension(msg, ext, field_value, json, opts.budget) case DescFieldValueEnum(): _read_enum_extension(msg, ext, field_value, json, opts) case DescFieldValueMessage(): @@ -209,12 +218,16 @@ def _read_extension( def _read_scalar_extension( - msg: Message, ext: DescExtension, field_value: DescFieldValueScalar, json: JsonValue + msg: Message, + ext: DescExtension, + field_value: DescFieldValueScalar, + json: JsonValue, + budget: Budget, ) -> None: if json is None: del msg[ext.type] else: - msg[ext.type] = _read_scalar(ext, field_value.scalar, json) + msg[ext.type] = _read_scalar(ext, field_value.scalar, json, budget) def _read_enum_extension( @@ -227,7 +240,9 @@ def _read_enum_extension( if _is_resetting_null(field_value.enum, json): del msg[ext.type] else: - value = _read_enum(field_value.enum, json, opts.ignore_unknown_fields) + value = _read_enum( + field_value.enum, json, opts.ignore_unknown_fields, opts.budget + ) if value is not None: msg[ext.type] = value @@ -242,6 +257,8 @@ def _read_message_extension( if _is_resetting_null(field_value.message, json): del msg[ext.type] else: + budget = opts.budget + budget.charge_message(field_value.message) value = field_value.message.type() _read_message(value, json, opts) msg[ext.type] = value @@ -258,21 +275,31 @@ def _read_list_extension( return if not isinstance(json, list): raise _field_error(ext, f"expected list got {type(json)}", TypeError) - msg[ext.type] = [ - v - for element in json - if (v := _read_container_item(ext, field_value.element, element, opts)) - is not None - ] + budget = opts.budget + budget.charge_list() + values = [] + for element in json: + v = _read_container_item(ext, field_value.element, element, opts) + if v is not None: + budget.charge_list_item() + values.append(v) + msg[ext.type] = values def _read_scalar_field( - msg: Message, field: DescField, field_value: DescFieldValueScalar, json: JsonValue + msg: Message, + field: DescField, + field_value: DescFieldValueScalar, + json: JsonValue, + budget: Budget, ) -> None: if json is None: msg._del_member(field) return - msg._set_member(field, _read_scalar(field, field_value.scalar, json)) + value = _read_scalar(field, field_value.scalar, json, budget) + if field_value.oneof is not None: + budget.charge_oneof() + msg._set_member(field, value) def _read_enum_field( @@ -285,8 +312,10 @@ def _read_enum_field( if _is_resetting_null(field_value.enum, json): msg._del_member(field) return - value = _read_enum(field_value.enum, json, opts.ignore_unknown_fields) + value = _read_enum(field_value.enum, json, opts.ignore_unknown_fields, opts.budget) if value is not None: + if field_value.oneof is not None: + opts.budget.charge_oneof() msg._set_member(field, value) @@ -301,12 +330,12 @@ def _read_list_field( return if not isinstance(json, list): raise _field_error(field, f"expected list got {type(json)}", TypeError) - list_.extend( - value - for element in json - if (value := _read_container_item(field, field_value.element, element, opts)) - is not None - ) + budget = opts.budget + for element in json: + value = _read_container_item(field, field_value.element, element, opts) + if value is not None: + budget.charge_list_item() + list_.append(value) def _read_map_field( @@ -320,10 +349,12 @@ def _read_map_field( return if not isinstance(json, dict): raise _field_error(field, f"expected dict got {type(json)}", TypeError) + budget = opts.budget for json_key, json_value in json.items(): - key = _read_map_key(field, field_value, json_key) + key = _read_map_key(field, field_value, json_key, budget) value = _read_container_item(field, field_value.value, json_value, opts) if value is not None: + budget.charge_dict_entry() dict_[key] = value @@ -337,17 +368,20 @@ def _read_message_field( if _is_resetting_null(field_value.message, json): msg._del_member(field) return - value = ( - msg._get_member(field) - if msg._contains_member(field) - else field_value.message.type() - ) + budget = opts.budget + if msg._contains_member(field): + value = msg._get_member(field) + else: + budget.charge_message(field_value.message) + value = field_value.message.type() _read_message(value, json, opts) + if field_value.oneof is not None: + budget.charge_oneof() msg._set_member(field, value) def _read_map_key( - field: DescField, field_value: DescFieldValueMap, json: JsonValue + field: DescField, field_value: DescFieldValueMap, json: JsonValue, budget: Budget ) -> bool | int | str: match field_value.key: case ScalarType.BOOL: @@ -357,13 +391,16 @@ def _read_map_key( return False raise _field_error(field, f"unexpected bool map key value {json}") case ScalarType.STRING: - return _read_string(field, json) + key = _read_string(field, json) + budget.charge_str(key) + return key case ( ScalarType.DOUBLE | ScalarType.FLOAT | ScalarType.BYTES ): # This is because the Map key is not narrow enough msg = f"invalid map key type: {field_value.key}" raise AssertionError(msg) case _: + budget.charge_int() return _read_int(field, field_value.key, json) @@ -374,13 +411,15 @@ def _read_container_item( opts: FromJsonOptions, ) -> bool | int | float | str | bytes | Message | Enum | None: if isinstance(element, ScalarType) and json is not None: - return _read_scalar(field, element, json) + return _read_scalar(field, element, json, opts.budget) if isinstance(element, DescMessage) and not _is_resetting_null(element, json): + budget = opts.budget + budget.charge_message(element) msg = element.type() _read_message(msg, json, opts) return msg if isinstance(element, DescEnum) and not _is_resetting_null(element, json): - return _read_enum(element, json, opts.ignore_unknown_fields) + return _read_enum(element, json, opts.ignore_unknown_fields, opts.budget) raise _field_error( field, f"unexpected null value for {'map value' if isinstance(field, DescField) and isinstance(field.value, DescFieldValueMap) else 'list item'}", @@ -391,6 +430,7 @@ def _read_enum( desc: DescEnum, json: JsonValue, ignore_unknown_fields: bool, # noqa: FBT001 + budget: Budget, ) -> Enum | None: if json is None: return desc.type(desc.values[0].number) @@ -402,6 +442,7 @@ def _read_enum( if ignore_unknown_fields: return None # Succeeds for open enum, raises an error for closed + budget.charge_enum() return desc.type(json) if isinstance(json, str): if value := desc._values_by_name.get(json): @@ -416,6 +457,17 @@ def _read_enum( def _read_scalar( + desc: DescField | DescExtension, + scalar_type: ScalarType, + json: JsonValue, + budget: Budget, +) -> bool | int | float | str | bytes: + value = _read_scalar_value(desc, scalar_type, json) + budget.charge_scalar(scalar_type, value) + return value + + +def _read_scalar_value( desc: DescField | DescExtension, scalar_type: ScalarType, json: JsonValue ) -> bool | int | float | str | bytes: match scalar_type: @@ -605,9 +657,13 @@ def _struct_from_json( assert isinstance(value_desc, DescMessage) # noqa: S101 value_wkt = match_wkt(value_desc) assert isinstance(value_wkt, WktValue) # noqa: S101 + budget = opts.budget for k, v in json.items(): + budget.charge_message(value_desc) val = cast("Value", value_desc.type()) _value_from_json(val, v, opts, value_wkt) + budget.charge_str(k) + budget.charge_dict_entry() msg.fields[k] = val @@ -624,9 +680,12 @@ def _list_value_from_json( assert isinstance(element_desc, DescMessage) # noqa: S101 element_wkt = match_wkt(element_desc) assert isinstance(element_wkt, WktValue) # noqa: S101 + budget = opts.budget for e in json: + budget.charge_message(element_desc) val = cast("Value", element_desc.type()) _value_from_json(val, e, opts, element_wkt) + budget.charge_list_item() msg.values.append(val) @@ -642,6 +701,8 @@ def _value_from_json( def _value_from_json_inner( msg: Value, json: JsonValue, opts: FromJsonOptions, wkt: WktValue ) -> None: + budget = opts.budget + budget.charge_oneof() match json: case None: msg.kind = Oneof( @@ -650,10 +711,13 @@ def _value_from_json_inner( case bool(): msg.kind = Oneof("bool_value", json) case int() | float(): + budget.charge_float() msg.kind = Oneof("number_value", float(json)) case str(): + budget.charge_str(json) msg.kind = Oneof("string_value", json) case list(): + budget.charge_message(wkt.list_value.message) lv_desc = wkt.list_value.message lv_wkt = match_wkt(lv_desc) assert isinstance(lv_wkt, WktListValue) # noqa: S101 @@ -661,6 +725,7 @@ def _value_from_json_inner( _list_value_from_json(lv, json, opts, lv_wkt.values) msg.kind = Oneof("list_value", lv) case dict(): + budget.charge_message(wkt.struct_value.message) struct_desc = wkt.struct_value.message struct_wkt = match_wkt(struct_desc) assert isinstance(struct_wkt, WktStruct) # noqa: S101 @@ -677,6 +742,7 @@ def message_from_json_value( *, ignore_unknown_fields: bool = False, registry: Registry | None = None, + allocation_limit: int | None = None, ) -> T: """Converts the Python value parsed from JSON data to a new Message of the given type. @@ -700,5 +766,8 @@ def message_from_json_value( ``` """ return message_type._from_json_value( - data, ignore_unknown_fields=ignore_unknown_fields, registry=registry + data, + ignore_unknown_fields=ignore_unknown_fields, + registry=registry, + allocation_limit=allocation_limit, ) diff --git a/src/protobuf/_message.py b/src/protobuf/_message.py index 91dadf9..4d6dabe 100644 --- a/src/protobuf/_message.py +++ b/src/protobuf/_message.py @@ -19,6 +19,7 @@ from typing_extensions import Buffer, TypeVar from . import _native_message +from ._budget import new_budget from ._descriptors import ( DescField, DescFieldValueEnum, @@ -582,6 +583,7 @@ def from_json( *, ignore_unknown_fields: bool = False, registry: Registry | None = None, + allocation_limit: int | None = None, ) -> Self: """Create a new message from a ProtoJSON string. @@ -594,6 +596,9 @@ def from_json( registry: This option is required to read `google.protobuf.Any` and extensions from JSON format. + allocation_limit: + If set, the approximate number of bytes of Python objects the + parse may allocate before raising a ValueError. Raises: json.JSONDecodeError: If json_source is not valid JSON. @@ -603,12 +608,21 @@ def from_json( """ msg = cls() msg._merge_from_json( - json, ignore_unknown_fields=ignore_unknown_fields, registry=registry + json, + ignore_unknown_fields=ignore_unknown_fields, + registry=registry, + allocation_limit=allocation_limit, ) return msg @classmethod - def from_binary(cls, data: Buffer, *, ignore_unknown_fields: bool = False) -> Self: + def from_binary( + cls, + data: Buffer, + *, + ignore_unknown_fields: bool = False, + allocation_limit: int | None = None, + ) -> Self: """Create a new message by parsing serialized binary data. To merge into an existing message, use [`merge_from_binary`][]. @@ -616,9 +630,16 @@ def from_binary(cls, data: Buffer, *, ignore_unknown_fields: bool = False) -> Se Args: data: Serialized binary protobuf data. Must not be mutated during parsing. ignore_unknown_fields: If `True`, unknown fields in the binary data are silently discarded. + allocation_limit: + If set, the approximate number of bytes of Python objects the + parse may allocate before raising a ValueError. """ message = cls() - message._merge_from_binary(data, ignore_unknown_fields=ignore_unknown_fields) + message._merge_from_binary( + data, + ignore_unknown_fields=ignore_unknown_fields, + allocation_limit=allocation_limit, + ) return message @classmethod @@ -667,8 +688,20 @@ def _resolve_field(self, key: str) -> DescField: # Marshaling methods overridden in native code when available. - def _merge_from_binary(self, data: Buffer, ignore_unknown_fields: bool) -> None: # noqa: FBT001 - opts = FromBinaryOptions(ignore_unknown_fields=ignore_unknown_fields) + def _merge_from_binary( + self, + data: Buffer, + ignore_unknown_fields: bool, # noqa: FBT001 + allocation_limit: int | None = None, + ) -> None: + budget = new_budget(allocation_limit) + # The root is charged here rather than in from_binary so both entry + # points share one budget semantic; a merge into an existing message + # overcharges by one base size. + budget.charge_message(self._desc) + opts = FromBinaryOptions( + ignore_unknown_fields=ignore_unknown_fields, budget=budget + ) view = memoryview(data) read_message(self, BinaryReader(view), opts, depth=0, length=len(view)) @@ -678,6 +711,7 @@ def _merge_from_json( *, ignore_unknown_fields: bool = False, registry: Registry | None = None, + allocation_limit: int | None = None, ) -> None: from json import loads as parse_json # noqa: PLC0415 @@ -685,8 +719,13 @@ def _merge_from_json( from ._from_json import FromJsonOptions, _read_message # noqa: PLC0415 json_value = parse_json(json) + budget = new_budget(allocation_limit) + # See _merge_from_binary for why the root is charged here. + budget.charge_message(self._desc) opts = FromJsonOptions( - ignore_unknown_fields=ignore_unknown_fields, registry=registry + ignore_unknown_fields=ignore_unknown_fields, + registry=registry, + budget=budget, ) _read_message(self, json_value, opts) @@ -726,16 +765,21 @@ def _from_json_value( *, ignore_unknown_fields: bool = False, registry: Registry | None = None, + allocation_limit: int | None = None, ) -> Self: # Needs to be lazy import since JSON specially handles many WKTs. from ._from_json import FromJsonOptions, _read_message # noqa: PLC0415 + budget = new_budget(allocation_limit) + budget.charge_message(cls._desc) message = cls() _read_message( message, data, FromJsonOptions( - ignore_unknown_fields=ignore_unknown_fields, registry=registry + ignore_unknown_fields=ignore_unknown_fields, + registry=registry, + budget=budget, ), ) return message diff --git a/src/protobuf/_unknown.py b/src/protobuf/_unknown.py index e9b5d42..4a18dad 100644 --- a/src/protobuf/_unknown.py +++ b/src/protobuf/_unknown.py @@ -79,11 +79,11 @@ def get_unknown_field( case DescFieldValueScalar(): reader = BinaryReader(memoryview(binary_fields[-1])) reader.tag() - return read_scalar(field_value.scalar, reader) + return read_scalar(field_value.scalar, reader, opts.budget) case DescFieldValueEnum(): reader = BinaryReader(memoryview(binary_fields[-1])) reader.tag() - enum_value = read_enum(field_value.enum, reader) + enum_value = read_enum(field_value.enum, reader, opts.budget) if isinstance(enum_value, Enum): return enum_value return None diff --git a/src/protobuf/_wkt_registry.py b/src/protobuf/_wkt_registry.py index 11e3d03..367a349 100644 --- a/src/protobuf/_wkt_registry.py +++ b/src/protobuf/_wkt_registry.py @@ -98,7 +98,7 @@ def to_json_value(self, msg: Message, _opts: ToJsonOptions) -> JsonValue: nanos_str = nanos_str[:6] return f"{iso_secs}.{nanos_str}Z" - def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bool: + def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> bool: from .wkt import Timestamp # noqa: PLC0415 value = cast("Timestamp", msg) @@ -128,6 +128,8 @@ def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bo " 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z inclusive" ) raise ValueError(err) + opts.budget.charge_int() + opts.budget.charge_int() value.seconds = Timestamp.from_datetime(dt).seconds value.nanos = nanos return True @@ -167,7 +169,7 @@ def to_json_value(self, msg: Message, _opts: ToJsonOptions) -> JsonValue: text = "-" + text return text + "s" - def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bool: + def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> bool: from .wkt._mixin._const import DURATION_SECONDS_MAX # noqa: PLC0415 value = cast("Duration", msg) @@ -187,6 +189,8 @@ def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bo nanos = int(duration_match[2] + "0" * (9 - len(duration_match[2]))) if seconds < 0 or duration_match[1] == "-0": nanos = -nanos + opts.budget.charge_int() + opts.budget.charge_int() value.seconds = seconds value.nanos = nanos return True @@ -245,6 +249,7 @@ def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> boo if not desc: err = f"cannot decode {Any._desc.type_name} from JSON: {type_url} is not in the type registry" raise ValueError(err) + opts.budget.charge_message(desc) message = desc.type() if _has_custom_json(desc) and "value" in json: _read_message(message, json["value"], opts) @@ -253,6 +258,8 @@ def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> boo del json["@type"] _read_message(message, json, opts) any_ = Any.pack(message) + opts.budget.charge_str(any_.type_url) + opts.budget.charge_bytes(len(any_.value)) value.type_url = any_.type_url value.value = any_.value return True @@ -280,7 +287,7 @@ def to_json_value(self, msg: Message, _opts: ToJsonOptions) -> JsonValue: parts.append(proto_camel_path) return ",".join(parts) - def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bool: + def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> bool: from ._names import proto_snake_case # noqa: PLC0415 value = cast("FieldMask", msg) @@ -293,7 +300,10 @@ def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bo if "_" in path: err = f"cannot decode {value._desc.type_name} from JSON: path names must be lowerCamelCase" raise ValueError(err) - value.paths.append(proto_snake_case(path)) + path = proto_snake_case(path) # noqa: PLW2901 + opts.budget.charge_str(path) + opts.budget.charge_list_item() + value.paths.append(path) return True def mixin(self) -> type | None: @@ -375,13 +385,15 @@ def to_json_value(self, msg: Message, _opts: ToJsonOptions) -> JsonValue: return _scalar_to_json_value(self.value.scalar, msg[self.field]) - def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bool: + def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> bool: from ._from_json import _read_scalar # noqa: PLC0415 if json is None: del msg[self.field] else: - msg[self.field] = _read_scalar(self.field, self.value.scalar, json) + msg[self.field] = _read_scalar( + self.field, self.value.scalar, json, opts.budget + ) return True def mixin(self) -> type | None: diff --git a/tests/test_budget.py b/tests/test_budget.py new file mode 100644 index 0000000..4f1b811 --- /dev/null +++ b/tests/test_budget.py @@ -0,0 +1,196 @@ +# Copyright (c) 2025-2026 Buf Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import sys +from typing import TYPE_CHECKING, Any, cast + +import pytest + +from protobuf._budget import ( + BYTES_OVERHEAD, + DICT_ENTRY_SIZE, + EMPTY_DICT_SIZE, + EMPTY_LIST_SIZE, + INT_SIZE, + LIST_SLOT_SIZE, + OBJECT_HEADER_SIZE, + SLOT_SIZE, + TrackingBudget, + _instance_size, + _message_alloc_size, + str_size, +) +from protobuf._native_message import NativeMessageClass +from protobuf._wire import BinaryWriter, WireType +from protobuf.wkt import FileDescriptorSet, Struct, Timestamp, Value + +from .gen.lists_pb import Lists +from .gen.maps_pb import Maps +from .gen.oneofs_pb import Oneofs +from .gen.scalars_pb import Scalars + +if TYPE_CHECKING: + from collections.abc import Callable + + from protobuf import Message + +_IS_CPYTHON = sys.implementation.name == "cpython" +_GIL_ENABLED = getattr(sys, "_is_gil_enabled", lambda: True)() +# The str header shrank in 3.12 and is larger again without the GIL. +_STR_SIZES_MATCH = _IS_CPYTHON and sys.version_info >= (3, 12) and _GIL_ENABLED + +_TEXTS = [ + pytest.param("", id="empty"), + pytest.param("a" * 50, id="ascii"), + pytest.param("é" * 50, id="latin1"), + pytest.param("あ" * 50, id="cjk"), + pytest.param("a" * 50 + "あ", id="ascii+cjk"), + pytest.param("😀" * 50, id="emoji"), + pytest.param("あ" * 50 + "😀", id="cjk+emoji"), +] + + +class _NoBasicsize: + """Simulates a PyPy type, by being an instantiated object that includes the mro of a type.""" + + def __init__(self, real: type) -> None: + self.__mro__ = real.__mro__ + + +def _slot_count(message_type: type) -> int: + slots = 0 + for cls in message_type.__mro__: + names = cls.__dict__.get("__slots__", ()) + names = (names,) if isinstance(names, str) else names + slots += sum(1 for n in names if n not in ("__dict__", "__weakref__")) + return slots + + +@pytest.mark.skipif( + NativeMessageClass is not None, reason="fallback only needed for pure Python" +) +@pytest.mark.skipif( + sys.implementation.name != "cpython", reason="no __basicsize__ to compare against" +) +@pytest.mark.parametrize( + "message_type", + [Scalars, Lists, Maps, Oneofs, Struct, Value, Timestamp, FileDescriptorSet], +) +def test_instance_size_fallback_matches_basicsize(message_type: type[Any]) -> None: + # Ensure the type (and its __slots__) exists before inspecting it. + message_type() + expected = message_type.__basicsize__ + if sys.version_info < (3, 12): + # Before managed weakrefs, the __weakref__ slot occupied a pointer + # inside the instance, which the fallback deliberately excludes. + expected -= SLOT_SIZE + if not getattr(sys, "_is_gil_enabled", lambda: True)(): + # Free-threaded builds fold the GC header into a 32-byte object + # header, which basicsize includes and the formula does not. + expected -= 16 + mt = _NoBasicsize(message_type) + assert not hasattr(mt, "__basicsize__") + assert _instance_size(cast("type", mt)) == expected + + +@pytest.mark.skipif( + sys.implementation.name != "pypy", reason="exercises the real fallback path" +) +def test_instance_size_fallback_on_pypy() -> None: + Scalars() + assert not hasattr(Scalars, "__basicsize__") + assert _instance_size(Scalars) == OBJECT_HEADER_SIZE + SLOT_SIZE * _slot_count( + Scalars + ) + + +def test_alloc_size_cached_on_descriptor() -> None: + desc = Scalars._desc + object.__setattr__(desc, "_alloc_size", None) + budget = TrackingBudget(sys.maxsize) + budget.charge_message(desc) + assert desc._alloc_size is not None + assert budget.current == desc._alloc_size + budget.charge_message(desc) + assert budget.current == 2 * desc._alloc_size + + +@pytest.mark.skipif(not _STR_SIZES_MATCH, reason="str layout differs") +@pytest.mark.parametrize("text", _TEXTS) +def test_str_size_matches_interpreter(text: str) -> None: + assert str_size(text) == sys.getsizeof(text) + + +def _assert_charged(parse: Callable[[int], Message], expected: int) -> None: + """Asserts that the parse charges exactly the expected amount.""" + parse(expected) + with pytest.raises(ValueError, match="allocation budget exceeded"): + parse(expected - 1) + + +def _parsers(message: Message) -> list[Callable[[int], Message]]: + """Parses of the message from binary, JSON text, and a JSON tree.""" + cls = type(message) + data = message.to_binary() + text = message.to_json() + tree = json.loads(text) + return [ + lambda limit: cls.from_binary(data, allocation_limit=limit), + lambda limit: cls.from_json(text, allocation_limit=limit), + lambda limit: cls._from_json_value(tree, allocation_limit=limit), + ] + + +@pytest.mark.parametrize("text", _TEXTS) +def test_repeated_string_charge(text: str) -> None: + count = 3 + message = Lists(string_list=[text] * count) + expected = _message_alloc_size(Lists._desc) + expected += count * (str_size(text) + LIST_SLOT_SIZE) + for parse in _parsers(message): + _assert_charged(parse, expected) + + +def test_string_map_charge() -> None: + entries = {f"キー{i}": f"value{i}😀" for i in range(3)} + message = Maps(string_to_string=entries) + expected = _message_alloc_size(Maps._desc) + expected += sum( + str_size(k) + str_size(v) + DICT_ENTRY_SIZE for k, v in entries.items() + ) + for parse in _parsers(message): + _assert_charged(parse, expected) + + +def test_unknown_field_charge() -> None: + def field(number: int, payload: bytes) -> bytes: + writer = BinaryWriter() + writer.tag(number, WireType.LENGTH_DELIMITED) + writer.bytes_(payload) + return writer.finish() + + fields = [field(200, b"x"), field(1000, b"y" * 10), field(1000, b"z")] + expected = _message_alloc_size(Scalars._desc) + EMPTY_DICT_SIZE + # Each distinct number has a list and a dict entry, and a key if it is + # above the small ints the interpreter shares. + expected += 2 * (EMPTY_LIST_SIZE + DICT_ENTRY_SIZE) + INT_SIZE + expected += sum(BYTES_OVERHEAD + len(f) + LIST_SLOT_SIZE for f in fields) + + data = b"".join(fields) + _assert_charged( + lambda limit: Scalars.from_binary(data, allocation_limit=limit), expected + ) diff --git a/tests/test_from_binary.py b/tests/test_from_binary.py index a18a2af..9466879 100644 --- a/tests/test_from_binary.py +++ b/tests/test_from_binary.py @@ -29,6 +29,7 @@ merge_from_binary, ) from protobuf._wire import BinaryWriter, WireType +from protobuf.wkt import FileDescriptorSet from .gen.delimited_encoding_pb import DelimitedEncoding from .gen.enums_pb import ClosedColor, EnumMessage @@ -463,3 +464,55 @@ def test_delimited_encoding(self) -> None: # Should be enough to check every field is there without asserting the content. assert msg._unknown_fields is not None assert {2, 3} == set(msg._unknown_fields.keys()) + + +class TestAllocationLimit: + def test_within_limit(self) -> None: + data = Scalars(string_field="a" * 1000).to_binary() + msg = Scalars.from_binary(data, allocation_limit=1024 * 1024) + assert msg.string_field == "a" * 1000 + + def test_string_exceeds_limit(self) -> None: + data = Scalars(string_field="a" * 1000).to_binary() + with pytest.raises(ValueError, match="allocation budget exceeded"): + Scalars.from_binary(data, allocation_limit=500) + + def test_repeated_exceeds_limit(self) -> None: + data = Lists(string_list=["a" * 100] * 100).to_binary() + with pytest.raises(ValueError, match="allocation budget exceeded"): + Lists.from_binary(data, allocation_limit=5000) + + def test_nested_messages_exceed_limit(self) -> None: + data = Lists(msg_list=[Lists.Msg() for _ in range(1000)]).to_binary() + with pytest.raises(ValueError, match="allocation budget exceeded"): + Lists.from_binary(data, allocation_limit=10_000) + + def test_unknown_fields_exceed_limit(self) -> None: + w = BinaryWriter() + w.tag(1000, WireType.LENGTH_DELIMITED) + w.bytes_(b"x" * 10_000) + data = w.finish() + with pytest.raises(ValueError, match="allocation budget exceeded"): + Scalars.from_binary(data, allocation_limit=5000) + + def test_no_limit_by_default(self) -> None: + data = Lists(string_list=["a" * 100] * 100).to_binary() + msg = Lists.from_binary(data) + assert len(msg.string_list) == 100 + + def test_merge_from_binary_limit(self) -> None: + data = Scalars(string_field="a" * 1000).to_binary() + msg = Scalars() + with pytest.raises(ValueError, match="allocation budget exceeded"): + merge_from_binary(msg, data, allocation_limit=500) + + def test_descriptor_set_amplification_rejected(self) -> None: + # Each empty FileDescriptorProto is two wire bytes (b"\x0a\x00": field 1, + # LEN, length 0) yet allocates a full descriptor instance with all its + # default containers -- ~700 bytes each, a ~350x blow-up. 1.9M entries + # is 3.8 MB on the wire but ~1.3 GB parsed. + rpc_read_limit = 4 * 1024 * 1024 + data = b"\x0a\x00" * 1_900_000 + assert len(data) < rpc_read_limit + with pytest.raises(ValueError, match="allocation budget exceeded"): + FileDescriptorSet.from_binary(data, allocation_limit=64 * 1024 * 1024) diff --git a/tests/test_json.py b/tests/test_json.py index f15c6b2..129e8fc 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -25,6 +25,7 @@ message_from_json_value, message_to_json_value, ) +from protobuf.wkt import FileDescriptorSet, Struct from .gen.enums_pb import ClosedColor, Color, EnumMessage from .gen.json_enum_names_pb import JsonEnumNames, Season @@ -824,3 +825,56 @@ def test_to_json_cyclic_message() -> None: match="exceeded maximum recursion depth 100 while serializing message", ): message_to_json_value(msg) + + +class TestAllocationLimit: + def test_within_limit(self) -> None: + data = Scalars(string_field="a" * 1000).to_json() + msg = Scalars.from_json(data, allocation_limit=1024 * 1024) + assert msg.string_field == "a" * 1000 + + def test_string_exceeds_limit(self) -> None: + data = Scalars(string_field="a" * 1000).to_json() + with pytest.raises(ValueError, match="allocation budget exceeded"): + Scalars.from_json(data, allocation_limit=500) + + def test_repeated_exceeds_limit(self) -> None: + data = Lists(string_list=["a" * 100] * 100).to_json() + with pytest.raises(ValueError, match="allocation budget exceeded"): + Lists.from_json(data, allocation_limit=5000) + + def test_nested_messages_exceed_limit(self) -> None: + data = Lists(msg_list=[Lists.Msg() for _ in range(1000)]).to_json() + with pytest.raises(ValueError, match="allocation budget exceeded"): + Lists.from_json(data, allocation_limit=10_000) + + def test_map_exceeds_limit(self) -> None: + data = Maps( + string_to_string={f"key{i}": "x" * 50 for i in range(100)} + ).to_json() + with pytest.raises(ValueError, match="allocation budget exceeded"): + Maps.from_json(data, allocation_limit=2000) + + def test_struct_exceeds_limit(self) -> None: + data = json.dumps({f"key{i}": "x" * 50 for i in range(100)}) + with pytest.raises(ValueError, match="allocation budget exceeded"): + Struct.from_json(data, allocation_limit=2000) + + def test_from_json_value_limit(self) -> None: + data = cast("dict[str, Any]", {"stringList": ["a" * 100] * 100}) + with pytest.raises(ValueError, match="allocation budget exceeded"): + message_from_json_value(Lists, data, allocation_limit=5000) + + def test_no_limit_by_default(self) -> None: + data = Lists(string_list=["a" * 100] * 100).to_json() + msg = Lists.from_json(data) + assert len(msg.string_list) == 100 + + def test_descriptor_set_amplification_rejected(self) -> None: + # Each empty "{}" in "file" for FileDescriptorSet + # parses to a full FileDescriptorProto (~700 bytes) from ~3 JSON bytes. + rpc_read_limit = 4 * 1024 * 1024 + data = '{"file":[' + ",".join(["{}"] * 1_200_000) + "]}" + assert len(data) < rpc_read_limit + with pytest.raises(ValueError, match="allocation budget exceeded"): + FileDescriptorSet.from_json(data, allocation_limit=64 * 1024 * 1024)