From b02e111de18853f71d5ddb51393dcd11276b9140 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Mon, 18 May 2026 17:05:00 +0200 Subject: [PATCH 01/44] feat: allow setting a power sensor in the flex_model of an asset when patching Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/storage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 807774fe5a..eaf8b69425 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -18,7 +18,7 @@ from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.units import QuantityField from flexmeasures.data.schemas.scheduling import metadata -from flexmeasures.data.schemas.sensors import VariableQuantityField +from flexmeasures.data.schemas.sensors import SensorIdField, VariableQuantityField from flexmeasures.utils.unit_utils import ( ur, is_power_unit, @@ -374,6 +374,10 @@ class DBStorageFlexModelSchema(Schema): Schema for flex-models stored in the db. Supports fixed quantities and sensor references, while disallowing time series specs. """ + sensor = SensorIdField( + required=False, + ) + soc_min = VariableQuantityField( to_unit="MWh", data_key="soc-min", From 3ffdaa00fdc0bcc9054979180943c6159e6599eb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 15:46:18 +0200 Subject: [PATCH 02/44] fix: fields without a data_key just use the variable name as data_key Signed-off-by: F.N. Claessen --- flexmeasures/ui/tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/ui/tests/test_utils.py b/flexmeasures/ui/tests/test_utils.py index 57f42bb166..09a68ff005 100644 --- a/flexmeasures/ui/tests/test_utils.py +++ b/flexmeasures/ui/tests/test_utils.py @@ -106,7 +106,7 @@ def test_ui_flexmodel_schema(): schema_keys = [] for value in DBStorageFlexModelSchema().fields.values(): - schema_keys.append(value.data_key) + schema_keys.append(value.data_key if value.data_key else value.name) schema_keys = set(schema_keys) ui_flexmodel_schema_fields = set(ui_flexmodel_schema_fields) From 4f5b116f9a610652bbeba3440f1e772863857f99 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 15:57:20 +0200 Subject: [PATCH 03/44] feat: switch to consumption and production sensor reference fields in DBStorageFlexModelSchema Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/storage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 2bb74626c7..fb266efed9 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -18,7 +18,10 @@ from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.units import QuantityField from flexmeasures.data.schemas.scheduling import metadata -from flexmeasures.data.schemas.sensors import SensorIdField, VariableQuantityField +from flexmeasures.data.schemas.sensors import ( + SensorReferenceSchema, + VariableQuantityField, +) from flexmeasures.utils.unit_utils import ( ur, is_power_unit, @@ -375,9 +378,8 @@ class DBStorageFlexModelSchema(Schema): Schema for flex-models stored in the db. Supports fixed quantities and sensor references, while disallowing time series specs. """ - sensor = SensorIdField( - required=False, - ) + consumption = fields.Nested(SensorReferenceSchema) + production = fields.Nested(SensorReferenceSchema) soc_min = VariableQuantityField( to_unit="MWh", From c3f2d68d5d5cc6c3b4667eefa0ff5e26bfb77a15 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 15:57:39 +0200 Subject: [PATCH 04/44] feat: add UI support for new fields Signed-off-by: F.N. Claessen --- .../data/schemas/scheduling/__init__.py | 18 ++++++++++++++++++ .../data/schemas/scheduling/metadata.py | 8 ++++++++ 2 files changed, 26 insertions(+) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 917a6ba990..09d5114783 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -582,6 +582,24 @@ def _to_currency_per_mwh(price_unit: str) -> str: } UI_FLEX_MODEL_SCHEMA: Dict[str, Dict[str, Any]] = { + "consumption": { + "default": None, + "description": rst_to_openapi(metadata.CONSUMPTION.description), + "types": { + "backend": "typeTwo", + "ui": "A sensor which records the scheduled consumption.", + }, + "example-units": EXAMPLE_UNIT_TYPES["power"], + }, + "production": { + "default": None, + "description": rst_to_openapi(metadata.PRODUCTION.description), + "types": { + "backend": "typeTwo", + "ui": "A sensor which records the scheduled production.", + }, + "example-units": EXAMPLE_UNIT_TYPES["power"], + }, "soc-min": { "default": None, "description": rst_to_openapi(metadata.SOC_MIN.description), diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 5852d6d286..4daebfbfba 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -185,6 +185,14 @@ def to_dict(self): # FLEX-MODEL +CONSUMPTION = MetaData( + description="Sensor used to record the scheduled consumption.", + example={"sensor": 14}, +) +PRODUCTION = MetaData( + description="Sensor used to record the scheduled production.", + example={"sensor": 15}, +) STATE_OF_CHARGE = MetaData( description="Sensor used to record the scheduled state of charge. If ``soc-at-start`` is omitted, FlexMeasures will also use this field to infer the starting state of charge. For this use case, the field may also contain a time series specification instead. When a sensor is used, its unit may be an energy unit (e.g. MWh or kWh) or a percentage (%). For sensors with a % unit, the ``soc-max`` flex-model field must be set to a non-zero value to allow converting between the energy-based schedule and a percentage.", example={"sensor": 12}, From 711e228228175b1f67e242648e3e27201b3847f2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 15:58:52 +0200 Subject: [PATCH 05/44] docs: document new fields in scheduling.rst Signed-off-by: F.N. Claessen --- documentation/features/scheduling.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 3801f2bda0..1643330a6d 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -182,7 +182,13 @@ For more details on the possible formats for field values, see :ref:`variable_qu * - Field - Example value - - Description + - Description + * - ``consumption`` + - |CONSUMPTION.example| + - .. include:: ../_autodoc/CONSUMPTION.rst + * - ``production`` + - |PRODUCTION.example| + - .. include:: ../_autodoc/PRODUCTION.rst * - ``state-of-charge`` - |STATE_OF_CHARGE.example| - .. include:: ../_autodoc/STATE_OF_CHARGE.rst From 1fa459e4766b96c6305aed9aea53034faf18b12e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 16:10:33 +0200 Subject: [PATCH 06/44] feat: add new fields to StorageFlexModelSchema Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/storage.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index fb266efed9..aa186edba3 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -85,6 +85,15 @@ class StorageFlexModelSchema(Schema): metadata=dict(description="ID of the asset that is requested to be scheduled."), ) + consumption = fields.Nested( + SensorReferenceSchema, + metadata=metadata.CONSUMPTION.to_dict(), + ) + production = fields.Nested( + SensorReferenceSchema, + metadata=metadata.PRODUCTION.to_dict(), + ) + soc_at_start = QuantityField( required=False, to_unit="MWh", From ed640b2db6e4e2c39619fb529c7d7516fe40bc4d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 16:17:16 +0200 Subject: [PATCH 07/44] chore: update openapi-specs.json Signed-off-by: F.N. Claessen --- flexmeasures/ui/static/openapi-specs.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 74153ec68b..f2bc2bedd0 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -5992,6 +5992,20 @@ "StorageFlexModelSchemaOpenAPI": { "type": "object", "properties": { + "consumption": { + "description": "Sensor used to record the scheduled consumption.", + "example": { + "sensor": 14 + }, + "$ref": "#/components/schemas/SensorReference" + }, + "production": { + "description": "Sensor used to record the scheduled production.", + "example": { + "sensor": 15 + }, + "$ref": "#/components/schemas/SensorReference" + }, "soc-at-start": { "type": "string", "description": "The (estimated) state of charge at the beginning of the schedule (for storage devices, this defaults to 0).\nUsually added to each scheduling request.\n", From 4733868d8843ebdc3063c29ea607741b1535ae59 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 16:44:02 +0200 Subject: [PATCH 08/44] =?UTF-8?q?storage:=20add=20consumption=20and=20prod?= =?UTF-8?q?uction=20output=20sensor=20support=20Context:=20-=20StorageSche?= =?UTF-8?q?duler=20could=20already=20write=20state-of-charge=20schedules?= =?UTF-8?q?=20to=20a=20secondary=20sensor=20-=20Users=20wanted=20the=20sam?= =?UTF-8?q?e=20for=20consumption=20and=20production=20power=20Change:=20-?= =?UTF-8?q?=20Add=20StorageScheduler.=5Fbuild=5Fconsumption=5Fproduction?= =?UTF-8?q?=5Fschedules()=20static=20method=20-=20Call=20it=20in=20compute?= =?UTF-8?q?(),=20with=20resampling=20and=20rounding=20matching=20the=20soc?= =?UTF-8?q?=5Fschedule=20pattern=20-=20If=20only=20consumption=20sensor=20?= =?UTF-8?q?defined:=20full=20power=20profile=20(consumption=20positive,=20?= =?UTF-8?q?production=20negative)=20-=20If=20only=20production=20sensor=20?= =?UTF-8?q?defined:=20full=20power=20profile=20inverted=20(production=20po?= =?UTF-8?q?sitive,=20consumption=20negative)=20-=20If=20both=20defined:=20?= =?UTF-8?q?split=20=E2=80=94=20non-negative=20part=20to=20consumption=20se?= =?UTF-8?q?nsor,=20sign-flipped=20non-positive=20part=20to=20production=20?= =?UTF-8?q?sensor=20-=20Include=20results=20in=20return=5Fmultiple=20outpu?= =?UTF-8?q?t=20as=20consumption=5Fschedule=20/=20production=5Fschedule=20e?= =?UTF-8?q?ntries=20-=20Sign=20convention=20is=20encoded=20in=20the=20key?= =?UTF-8?q?=20name=20so=20no=20consumption=5Fis=5Fpositive=20attribute=20i?= =?UTF-8?q?s=20needed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flexmeasures/data/models/planning/storage.py | 118 ++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 93a67a3e96..93db661796 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1567,6 +1567,83 @@ def _build_soc_schedule( ) return soc_schedule + @staticmethod + def _build_consumption_production_schedules( + flex_model: list[dict], + ems_schedule: pd.DataFrame, + ) -> dict: + """Build consumption and/or production power schedules for devices that define output sensors. + + Each device's flex model may define a ``consumption`` sensor, a ``production`` sensor, or both. + The schedule stored on each sensor depends on which sensors are defined: + + - **Only** ``consumption`` **sensor defined**: the full power schedule is written to that + sensor using the standard FlexMeasures sign convention (consumption positive, production + negative). + - **Only** ``production`` **sensor defined**: the full power schedule is written to that + sensor with the sign inverted (production positive, consumption negative). + - **Both** ``consumption`` **and** ``production`` **sensors defined**: only the non-negative + part of the schedule is written to the consumption sensor, and only the non-positive part + (sign-flipped to positive values) is written to the production sensor. + + Because the sign convention is encoded in the sensor key name (``consumption`` vs. + ``production``), these sensors do not need a ``consumption_is_positive`` attribute. + + Unit conversion from MW to each sensor's unit is applied. + + :param flex_model: List of per-device flex models (after deserialization). + :param ems_schedule: DataFrame of per-device power schedules in MW (consumption positive). + :returns: Dict mapping each output sensor to its power schedule. + """ + schedules: dict = {} + for d, flex_model_d in enumerate(flex_model): + consumption_field = flex_model_d.get("consumption") + production_field = flex_model_d.get("production") + consumption_sensor = ( + consumption_field["sensor"] + if isinstance(consumption_field, dict) and "sensor" in consumption_field + else None + ) + production_sensor = ( + production_field["sensor"] + if isinstance(production_field, dict) and "sensor" in production_field + else None + ) + if consumption_sensor is None and production_sensor is None: + continue + power_series = ems_schedule[d] # in MW; consumption is positive + if consumption_sensor is not None and production_sensor is None: + # Full power profile on the consumption sensor (consumption positive, production negative). + schedules[consumption_sensor] = convert_units( + power_series, + "MW", + consumption_sensor.unit, + event_resolution=consumption_sensor.event_resolution, + ) + elif production_sensor is not None and consumption_sensor is None: + # Full power profile on the production sensor (production positive, consumption negative). + schedules[production_sensor] = convert_units( + -power_series, + "MW", + production_sensor.unit, + event_resolution=production_sensor.event_resolution, + ) + else: + # Both sensors defined: split into non-negative (consumption) and non-positive (production) parts. + schedules[consumption_sensor] = convert_units( + power_series.clip(lower=0), + "MW", + consumption_sensor.unit, + event_resolution=consumption_sensor.event_resolution, + ) + schedules[production_sensor] = convert_units( + (-power_series).clip(lower=0), + "MW", + production_sensor.unit, + event_resolution=production_sensor.event_resolution, + ) + return schedules + def compute(self, skip_validation: bool = False) -> SchedulerOutputType: """Schedule a battery or Charge Point based directly on the latest beliefs regarding market prices within the specified time window. For the resulting consumption schedule, consumption is defined as positive values. @@ -1640,6 +1717,10 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: flex_model, ems_schedule, soc_at_start, device_constraints, resolution ) + consumption_production_schedule = self._build_consumption_production_schedules( + flex_model, ems_schedule + ) + # Resample each device schedule to the resolution of the device's power sensor if self.resolution is None: storage_schedule = { @@ -1649,6 +1730,12 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: for sensor in storage_schedule.keys() if sensor is not None } + consumption_production_schedule = { + sensor: consumption_production_schedule[sensor] + .resample(sensor.event_resolution) + .mean() + for sensor in consumption_production_schedule.keys() + } # Round schedule if self.round_to_decimals: @@ -1661,6 +1748,12 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: sensor: soc_schedule[sensor].round(self.round_to_decimals) for sensor in soc_schedule.keys() } + consumption_production_schedule = { + sensor: consumption_production_schedule[sensor].round( + self.round_to_decimals + ) + for sensor in consumption_production_schedule.keys() + } if self.return_multiple: storage_schedules = [ @@ -1694,7 +1787,30 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: } for sensor, soc in soc_schedule.items() ] - return storage_schedules + commitment_costs + soc_schedules + # Determine which sensors are consumption vs. production output sensors + consumption_output_sensors = { + flex_model_d["consumption"]["sensor"] + for flex_model_d in flex_model + if isinstance(flex_model_d.get("consumption"), dict) + and "sensor" in flex_model_d["consumption"] + } + consumption_production_schedules = [ + { + "name": "consumption_schedule" + if sensor in consumption_output_sensors + else "production_schedule", + "data": data, + "sensor": sensor, + "unit": sensor.unit, + } + for sensor, data in consumption_production_schedule.items() + ] + return ( + storage_schedules + + commitment_costs + + soc_schedules + + consumption_production_schedules + ) else: return storage_schedule[sensors[0]] From a96edd484ec6a19940ef2a30c38027620c19d5c8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 16:50:30 +0200 Subject: [PATCH 09/44] =?UTF-8?q?tests:=20expand=20storage=20scheduler=20t?= =?UTF-8?q?ests=20for=20consumption/production=20output=20sensors=20Contex?= =?UTF-8?q?t:=20-=20StorageScheduler=20now=20supports=20writing=20schedule?= =?UTF-8?q?s=20to=20consumption=20and=20production=20sensors=20Change:=20-?= =?UTF-8?q?=20test=5Fbattery=5Fsolver=5Fmulti=5Fcommitment:=20add=20consum?= =?UTF-8?q?ption=20and=20production=20output=20sensors=20=20=20to=20the=20?= =?UTF-8?q?battery,=20include=20them=20in=20the=20flex-model,=20and=20veri?= =?UTF-8?q?fy=20unit=20conversion=20(MW=20=E2=86=92=20kW)=20=20=20and=20th?= =?UTF-8?q?e=20split=20logic=20(all-positive=20schedule=20=E2=86=92=20cons?= =?UTF-8?q?umption=20all=20positive,=20production=20all=20zero)=20-=20test?= =?UTF-8?q?=5Ftrigger=5Fschedule=5Fuses=5Fstate=5Fof=5Fcharge=5Fsensor=5Ff?= =?UTF-8?q?or=5Fsoc=5Fat=5Fstart:=20add=20production=20=20=20output=20sens?= =?UTF-8?q?or=20and=20verify=2096=20beliefs=20are=20stored=20after=20sched?= =?UTF-8?q?uling=20-=20test=5Fadd=5Fstorage=5Fschedule=5Fuses=5Fstate=5Fof?= =?UTF-8?q?=5Fcharge=5Fsensor=5Ffor=5Fsoc=5Fat=5Fstart:=20add=20consumptio?= =?UTF-8?q?n=20=20=20output=20sensor=20and=20verify=2048=20beliefs=20are?= =?UTF-8?q?=20stored=20after=20scheduling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_sensor_schedules_fresh_db.py | 25 +++++++++++- .../cli/tests/test_data_add_fresh_db.py | 21 ++++++++++ .../models/planning/tests/test_storage.py | 38 +++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index 200243b1fe..853fa8e3c1 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -235,14 +235,26 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( event_value=50, ) ) - fresh_db.session.commit() + # Also add a production output sensor to verify the production schedule is stored. + # (Using only a production sensor means the full power profile is stored with sign inverted: + # production positive, consumption negative.) sensor = ( Sensor.query.filter(Sensor.name == "power") .join(GenericAsset, GenericAsset.id == Sensor.generic_asset_id) .filter(GenericAsset.name == "Test battery") .one_or_none() ) + production_output_sensor = Sensor( + name="production output", + generic_asset=sensor.generic_asset, + unit="MW", + event_resolution=sensor.event_resolution, + ) + fresh_db.session.add(production_output_sensor) + fresh_db.session.commit() + + message["flex-model"]["production"] = {"sensor": production_output_sensor.id} with app.test_client() as client: trigger_schedule_response = client.post( @@ -268,6 +280,17 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( sensor = fresh_db.session.get(Sensor, sensor.id) assert sensor.generic_asset.get_attribute("soc_in_mwh") == pytest.approx(0.02) + # Verify the production output sensor received schedule data. + # Only the production sensor is defined (no consumption sensor), so the full power profile + # is stored with inverted sign: production as positive, consumption as negative. + production_output_sensor = fresh_db.session.get( + Sensor, production_output_sensor.id + ) + production_beliefs = production_output_sensor.search_beliefs( + event_starts_after=parse_datetime(message["start"]) + ) + assert len(production_beliefs) == 96 # 24h at 15-min resolution + @pytest.mark.parametrize( "context_sensor, asset_sensor, parent_sensor, expect_sensor", diff --git a/flexmeasures/cli/tests/test_data_add_fresh_db.py b/flexmeasures/cli/tests/test_data_add_fresh_db.py index 6abc036585..12a85505ab 100644 --- a/flexmeasures/cli/tests/test_data_add_fresh_db.py +++ b/flexmeasures/cli/tests/test_data_add_fresh_db.py @@ -488,6 +488,17 @@ def test_add_storage_schedule_uses_state_of_charge_sensor_for_soc_at_start( belief_time=datetime.fromisoformat(start), ) ) + + # Add a consumption output sensor to verify the full power profile is stored on it + # (only the consumption sensor is defined, so the sign convention is consumption positive, + # production negative). + consumption_output_sensor = Sensor( + name="consumption output", + generic_asset=charging_station, + unit="MW", + event_resolution=power_sensor.event_resolution, + ) + fresh_db.session.add(consumption_output_sensor) fresh_db.session.commit() cli_input_params = { @@ -504,6 +515,7 @@ def test_add_storage_schedule_uses_state_of_charge_sensor_for_soc_at_start( "soc-min": "0 MWh", "soc-max": "5 MWh", "power-capacity": "2 MW", + "consumption": {"sensor": consumption_output_sensor.id}, } ), } @@ -513,3 +525,12 @@ def test_add_storage_schedule_uses_state_of_charge_sensor_for_soc_at_start( check_command_ran_without_error(result) assert len(power_sensor.search_beliefs()) == 48 assert power_sensor.generic_asset.get_attribute("soc_in_mwh") == 2.5 + + # Verify the consumption output sensor received the full power schedule. + # A charging station is consumption-only (non-negative), so the full schedule + # is non-negative and equals what is stored on the power sensor. + consumption_output_sensor = fresh_db.session.get( + Sensor, consumption_output_sensor.id + ) + assert len(consumption_output_sensor.search_beliefs()) == 48 + diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index dbb5d792f7..16e2608643 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -29,6 +29,24 @@ def test_battery_solver_multi_commitment(add_battery_assets, db): index = initialize_index(start=start, end=end, resolution=resolution) production_prices = pd.Series(90, index=index) consumption_prices = pd.Series(100, index=index) + + # Add consumption and production output sensors to the battery asset + consumption_output_sensor = Sensor( + name="consumption output", + generic_asset=battery.generic_asset, + unit="kW", + event_resolution=resolution, + ) + production_output_sensor = Sensor( + name="production output", + generic_asset=battery.generic_asset, + unit="kW", + event_resolution=resolution, + ) + db.session.add(consumption_output_sensor) + db.session.add(production_output_sensor) + db.session.flush() + scheduler: Scheduler = StorageScheduler( battery, start, @@ -37,6 +55,8 @@ def test_battery_solver_multi_commitment(add_battery_assets, db): flex_model={ "soc-at-start": f"{soc_at_start} MWh", "soc-min": "0 MWh", + "consumption": {"sensor": consumption_output_sensor.id}, + "production": {"sensor": production_output_sensor.id}, "soc-max": "1 MWh", "power-capacity": "1 MVA", "soc-minima": [ @@ -131,6 +151,24 @@ def test_battery_solver_multi_commitment(add_battery_assets, db): costs["a sample commitment penalizing demand/supply"], 1 * (1 - 0.4) ) + # Check consumption/production output sensor schedules. + # The battery charges at a constant rate (all positive values), so the consumption schedule + # should match the power schedule in kW, and the production schedule should be all zeros. + consumption_result = next( + r for r in results if r.get("name") == "consumption_schedule" + ) + production_result = next( + r for r in results if r.get("name") == "production_schedule" + ) + assert consumption_result["sensor"] is consumption_output_sensor + assert consumption_result["unit"] == "kW" + assert production_result["sensor"] is production_output_sensor + assert production_result["unit"] == "kW" + # Both sensors have the same resolution as the power sensor, so no resampling occurs. + expected_kw = (1 - 0.4) / 24 * 1000 # MW -> kW + np.testing.assert_allclose(consumption_result["data"], expected_kw) + np.testing.assert_allclose(production_result["data"], 0) + def test_battery_relaxation(add_battery_assets, db): """Check that resolving SoC breaches is more important than resolving device power breaches. From e63ee437abf815ccdcd8f0a82766e63f13b6a114 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 16:51:15 +0200 Subject: [PATCH 10/44] docs: document consumption/production output sensor semantics and add changelog entry Context: - StorageScheduler now writes schedules to consumption/production sensors Change: - Expand CONSUMPTION and PRODUCTION metadata descriptions with the split logic (only consumption, only production, or both defined) and clarify that the sign convention is encoded in the key name (no consumption_is_positive attribute needed) - Add changelog entry in v0.33.0 New features section (PR number TBD) --- documentation/changelog.rst | 1 + .../data/schemas/scheduling/metadata.py | 22 +++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 4a760c37ff..8d91db102c 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -23,6 +23,7 @@ New features * Improve UX after deleting a child asset through the UI [see `PR #2119 `_] * Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 `_ and `PR #2151 `_] * Support sensor references for efficiency fields in storage flex-models [see `PR #2142 `_] +* The ``consumption`` and ``production`` flex-model fields for the ``StorageScheduler`` now act as output sensors: the scheduler writes the resulting power schedule to those sensors, with unit conversion and resampling applied. When only one of the two is defined, the full power profile is written (sign-inverted for the production sensor). When both are defined, the schedule is split into its non-negative (consumption) and non-positive (production) parts [see `PR #XXXX `_] * Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] * Add support for filtering sensor data GET requests by ``source-type`` on ``/api/v3_0/sensors//data`` [see `PR #2127 `_] diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 4daebfbfba..b64fe14d07 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -186,11 +186,29 @@ def to_dict(self): CONSUMPTION = MetaData( - description="Sensor used to record the scheduled consumption.", + description="""Sensor used to record the scheduled power as seen from a consumption perspective. + +The sign convention is determined by the key name, so the sensor itself does not need a ``consumption_is_positive`` attribute. + +Depending on which output sensors are defined: + +- **Only** ``consumption`` **defined**: the full power schedule is stored on this sensor using the + standard FlexMeasures sign convention (consumption positive, production negative). +- **Only** ``production`` **defined**: the full power schedule is stored on the production sensor + with the sign inverted (production positive, consumption negative). +- **Both defined**: only the non-negative part of the schedule is stored on this sensor (zero for + time steps with net production), and only the non-positive part (sign-flipped) is stored on the + production sensor. +""", example={"sensor": 14}, ) PRODUCTION = MetaData( - description="Sensor used to record the scheduled production.", + description="""Sensor used to record the scheduled power as seen from a production perspective. + +The sign convention is determined by the key name, so the sensor itself does not need a ``consumption_is_positive`` attribute. + +See ``consumption`` for the full description of the split logic when both sensors are defined. +""", example={"sensor": 15}, ) STATE_OF_CHARGE = MetaData( From 82c6015b3608ba8efef46aa0795099cb019eb80c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 20 May 2026 17:02:47 +0200 Subject: [PATCH 11/44] style: black Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_sensor_schedules_fresh_db.py | 4 +--- flexmeasures/cli/tests/test_data_add_fresh_db.py | 1 - flexmeasures/data/models/planning/storage.py | 8 +++++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index 853fa8e3c1..c60626b13e 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -283,9 +283,7 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( # Verify the production output sensor received schedule data. # Only the production sensor is defined (no consumption sensor), so the full power profile # is stored with inverted sign: production as positive, consumption as negative. - production_output_sensor = fresh_db.session.get( - Sensor, production_output_sensor.id - ) + production_output_sensor = fresh_db.session.get(Sensor, production_output_sensor.id) production_beliefs = production_output_sensor.search_beliefs( event_starts_after=parse_datetime(message["start"]) ) diff --git a/flexmeasures/cli/tests/test_data_add_fresh_db.py b/flexmeasures/cli/tests/test_data_add_fresh_db.py index 12a85505ab..66c81e8f9c 100644 --- a/flexmeasures/cli/tests/test_data_add_fresh_db.py +++ b/flexmeasures/cli/tests/test_data_add_fresh_db.py @@ -533,4 +533,3 @@ def test_add_storage_schedule_uses_state_of_charge_sensor_for_soc_at_start( Sensor, consumption_output_sensor.id ) assert len(consumption_output_sensor.search_beliefs()) == 48 - diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 93db661796..8539e8e8bc 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1796,9 +1796,11 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: } consumption_production_schedules = [ { - "name": "consumption_schedule" - if sensor in consumption_output_sensors - else "production_schedule", + "name": ( + "consumption_schedule" + if sensor in consumption_output_sensors + else "production_schedule" + ), "data": data, "sensor": sensor, "unit": sensor.unit, From 4b1ab4f4240c863801b8209549127f1178f8abb7 Mon Sep 17 00:00:00 2001 From: saerts-gp Date: Thu, 2 Apr 2026 13:11:29 +0200 Subject: [PATCH 12/44] fix: guard sensor_d against None in StorageScheduler._prepare Fixes #2084 Context: - StorageScheduler._prepare crashes with AttributeError when a device in the asset tree has no sensor in its flex-model (only power-capacity) - Lines 672 and 740 already guard sensor_d against None, but line 902 was missed Change: - Add 'sensor_d is not None' check before accessing event_resolution - Matches the existing pattern used elsewhere in the same method Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 1 + flexmeasures/data/models/planning/storage.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 5c92a9b346..478aca05b9 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -44,6 +44,7 @@ Infrastructure / Support Bugfixes ----------- +* Fix ``StorageScheduler`` crash (``AttributeError: 'NoneType' object has no attribute 'event_resolution'``) when scheduling a site whose asset tree contains non-storage devices with only a ``power-capacity`` in their ``flex-model`` (no ``sensor`` key) [see `issue #2084 `_] * Fix DST transition handling by supporting both native Python ``datetime`` and pandas ``Timestamp`` objects in time series segment processing, preventing ``AttributeError`` when processing segments with differing UTC offsets [see `PR #2197 `_] * Fix forecasting regressor filtering to use only regressor beliefs known at the forecast ``belief_time`` [see `PR #2134 `_] * Make the sensor page forecast button train from available sensor history instead of the default 30-day window [see `PR #2187 `_] diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 8539e8e8bc..3f0202369b 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -906,7 +906,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[d]["efficiency"] = storage_efficiency[d] # Convert efficiency from sensor resolution to scheduling resolution - if sensor_d.event_resolution != timedelta(0): + if sensor_d is not None and sensor_d.event_resolution != timedelta(0): device_constraints[d]["efficiency"] **= ( resolution / sensor_d.event_resolution ) From a8a3e9e5c52d5acf280f2559293c4b79500a4983 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 16:05:14 +0200 Subject: [PATCH 13/44] feat: resample from the resolution of the storage-efficiency sensor to the scheduling resolution Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 3f0202369b..0a60eef871 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -906,7 +906,13 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[d]["efficiency"] = storage_efficiency[d] # Convert efficiency from sensor resolution to scheduling resolution - if sensor_d is not None and sensor_d.event_resolution != timedelta(0): + if isinstance(storage_efficiency[d], Sensor): + # Resample from the resolution of the storage-efficiency sensor + device_constraints[d]["efficiency"] **= ( + resolution / storage_efficiency[d].event_resolution + ) + elif sensor_d is not None and sensor_d.event_resolution != timedelta(0): + # Resample from the resolution of the power sensor device_constraints[d]["efficiency"] **= ( resolution / sensor_d.event_resolution ) From 1546e51d0b2dff862808203a839482490eff0fa0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 16:13:01 +0200 Subject: [PATCH 14/44] docs: extend field description accordingly Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index d91b3e8af4..2df2201388 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -315,8 +315,10 @@ def to_dict(self): As a result, each time step the energy is held longer leads to higher losses. This setting is crucial to some sorts of energy storage, e.g. thermal buffers. To give an example, when this setting is at 95% (or 0.95), this means a loss of 5% per time step. Defaults to 100% (no storage loss over time). -Note that the storage efficiency used by the scheduler is applied over each time step equal to the sensor resolution. +Note that the storage efficiency used by the scheduler is applied over each time step equal to the scheduling resolution. For example, a storage efficiency of 95 percent per (absolute) day, for scheduling a 1-hour resolution sensor, should be passed as a storage efficiency of :math:`0.95^{1/24} = 0.997865`. +Alternatively, to let FlexMeasures handle the conversion for you, record the storage-efficiency on a dedicated sensor (in this example, with a 24-hour event resolution). +Then reference that sensor in the storage-efficiency field. """, example="99.9%", ) From d42f65f175d188c49f55088c15c551b1f9266c6c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 16:26:18 +0200 Subject: [PATCH 15/44] fix: remove obsolete validator Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/storage.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index aa186edba3..80b9391c4d 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -325,19 +325,6 @@ def validate_asset(self, asset: Asset, **kwargs): if self.sensor is not None and self.sensor.asset != asset: raise ValidationError("Sensor does not belong to asset.") - @validates("storage_efficiency") - def validate_storage_efficiency_resolution( - self, unit: Sensor | ur.Quantity, **kwargs - ): - if ( - self.sensor is not None - and isinstance(unit, Sensor) - and unit.event_resolution != self.sensor.event_resolution - ): - raise ValidationError( - "Event resolution of the storage efficiency and the power sensor don't match. Resampling the storage efficiency is not supported." - ) - @validates_schema def check_redundant_efficiencies(self, data: dict, **kwargs): """ From 9bd75edb105d684d3d1b41144a6c07f848d051ac Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 16:27:54 +0200 Subject: [PATCH 16/44] feat: do not allow storage-efficiency to be set without a known resolution Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 16 ++++++++++++++++ flexmeasures/data/schemas/scheduling/storage.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 0a60eef871..f724e02fef 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -144,6 +144,8 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ] soc_gain = [flex_model_d.get("soc_gain") for flex_model_d in flex_model] soc_usage = [flex_model_d.get("soc_usage") for flex_model_d in flex_model] + consumption = [flex_model_d.get("consumption") for flex_model_d in flex_model] + production = [flex_model_d.get("production") for flex_model_d in flex_model] consumption_capacity = [ flex_model_d.get("consumption_capacity") for flex_model_d in flex_model ] @@ -916,6 +918,20 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[d]["efficiency"] **= ( resolution / sensor_d.event_resolution ) + elif isinstance(consumption[d], Sensor) and consumption[ + d + ].event_resolution != timedelta(0): + # Resample from the resolution of the consumption sensor + device_constraints[d]["efficiency"] **= ( + resolution / consumption[d].event_resolution + ) + elif isinstance(production[d], Sensor) and production[ + d + ].event_resolution != timedelta(0): + # Resample from the resolution of the production sensor + device_constraints[d]["efficiency"] **= ( + resolution / production[d].event_resolution + ) # check that storage constraints are fulfilled if not skip_validation: diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 80b9391c4d..183565c4b5 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -509,6 +509,22 @@ def __init__(self, *args, **kwargs): for field in self.declared_fields } + @validates("storage_efficiency") + def validate_storage_efficiency_resolution( + self, unit: Sensor | ur.Quantity, **kwargs + ): + if ( + isinstance(unit, ur.Quantity) + and not isinstance(self.consumption, Sensor) + and not isinstance(self.production, Sensor) + ): + raise ValidationError( + "The storage-efficiency cannot be interpreted without a resolution. " + "Record the storage-efficiency on a sensor instead (with a non-zero resolution) and then reference that sensor in the flex-model. " + "Alternatively, set the consumption or production field in the flex-model to reference a sensor, " + "and the scheduler will assume their resolution is the one to use." + ) + @validates_schema def forbid_time_series_specs(self, data: dict, **kwargs): """Do not allow time series specs for the flex-model fields saved in the db.""" From d6e15b65b47595a56106a7cc47a708107db2de99 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 17:06:35 +0200 Subject: [PATCH 17/44] docs: fill in PR number Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 2c5f0ad701..74f5cc8866 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -23,7 +23,7 @@ New features * Improve UX after deleting a child asset through the UI [see `PR #2119 `_] * Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 `_ and `PR #2151 `_] * Support sensor references for efficiency fields in storage flex-models [see `PR #2142 `_] -* The ``consumption`` and ``production`` flex-model fields for the ``StorageScheduler`` now act as output sensors: the scheduler writes the resulting power schedule to those sensors, with unit conversion and resampling applied. When only one of the two is defined, the full power profile is written (sign-inverted for the production sensor). When both are defined, the schedule is split into its non-negative (consumption) and non-positive (production) parts [see `PR #XXXX `_] +* The ``consumption`` and ``production`` flex-model fields for the ``StorageScheduler`` now act as output sensors: the scheduler writes the resulting power schedule to those sensors, with unit conversion and resampling applied. When only one of the two is defined, the full power profile is written (sign-inverted for the production sensor). When both are defined, the schedule is split into its non-negative (consumption) and non-positive (production) parts [see `PR #2190 `_] * Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] * Add support for filtering sensor data GET requests by ``source-type`` on ``/api/v3_0/sensors//data`` [see `PR #2127 `_] From 75fa2eb75c85e81627c0b2ce5d5cfa91015d98dc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 17:08:01 +0200 Subject: [PATCH 18/44] docs: API changelog entry Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 39414fd269..b11ff609d8 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -11,6 +11,7 @@ v3.0-31 | 2026-05-20 - Added a unified job status endpoint ``GET /api/v3_0/jobs/`` that returns the current execution status and a human-readable result message for any background job (scheduling, forecasting, etc.) identified by its UUID. - Switched from ``force_new_job_creation`` to ``force-new-job-creation`` (maintaining backwards compatibility) and added the field to `/assets/(id)/schedules/trigger` (POST) endpoint, too. - Support both snake_case and kebab-case fields in `/sensors//data` (GET), while only documenting the kebab-case ones. +- The ``consumption`` and ``production`` flex-model fields for the ``StorageScheduler`` now act as output sensors: the scheduler writes the resulting power schedule to those sensors, with unit conversion and resampling applied. When only one of the two is defined, the full power profile is written (sign-inverted for the production sensor). When both are defined, the schedule is split into its non-negative (consumption) and non-positive (production) parts. v3.0-30 | 2026-04-15 """""""""""""""""""" From 9f173fa182a530b52f45ec79cfdc788a689ece9e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 17:08:25 +0200 Subject: [PATCH 19/44] docs: simplify main changelog entry Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 74f5cc8866..07465010d3 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -23,7 +23,7 @@ New features * Improve UX after deleting a child asset through the UI [see `PR #2119 `_] * Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 `_ and `PR #2151 `_] * Support sensor references for efficiency fields in storage flex-models [see `PR #2142 `_] -* The ``consumption`` and ``production`` flex-model fields for the ``StorageScheduler`` now act as output sensors: the scheduler writes the resulting power schedule to those sensors, with unit conversion and resampling applied. When only one of the two is defined, the full power profile is written (sign-inverted for the production sensor). When both are defined, the schedule is split into its non-negative (consumption) and non-positive (production) parts [see `PR #2190 `_] +* Introduce the ``consumption`` and ``production`` flex-model fields for the ``StorageScheduler`` to save schedules to [see `PR #2190 `_] * Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] * Add support for filtering sensor data GET requests by ``source-type`` on ``/api/v3_0/sensors//data`` [see `PR #2127 `_] From 1033baf015d2b9ab4b4608f46415d276e60045f0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 17:19:40 +0200 Subject: [PATCH 20/44] feat: add more explicit checks in test Signed-off-by: F.N. Claessen --- .../v3_0/tests/test_sensor_schedules_fresh_db.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index c60626b13e..b6cc2989dd 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -215,7 +215,8 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( keep_scheduling_queue_empty, requesting_user, ): - message = message_for_trigger_schedule(resolution="PT1H") + scheduling_resolution = "PT1H" + message = message_for_trigger_schedule(resolution=scheduling_resolution) message["flex-context"] = { "consumption-price": {"sensor": add_market_prices_fresh_db["epex_da"].id}, "production-price": {"sensor": add_market_prices_fresh_db["epex_da"].id}, @@ -275,7 +276,8 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( ) assert get_soc_schedule_response.status_code == 200 assert get_soc_schedule_response.json["unit"] == "%" - assert get_soc_schedule_response.json["values"][0] == 50 + soc_values = get_soc_schedule_response.json["values"] + assert soc_values[0] == 50 sensor = fresh_db.session.get(Sensor, sensor.id) assert sensor.generic_asset.get_attribute("soc_in_mwh") == pytest.approx(0.02) @@ -288,6 +290,13 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( event_starts_after=parse_datetime(message["start"]) ) assert len(production_beliefs) == 96 # 24h at 15-min resolution + assert ( + soc_values[1] - soc_values[0] == -50 + ), "expected SoC to go down in the first hour" + assert ( + production_beliefs.resample_events(scheduling_resolution).iloc[0].event_value + > 0 + ), "expected first hour to show positive production" @pytest.mark.parametrize( From 061389bc8d50981f5696e65290e0d53fa3c8acca Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 17:31:43 +0200 Subject: [PATCH 21/44] data/services/scheduling: fix double sign inversion for consumption/production output schedules The StorageScheduler already applies correct sign conventions when returning consumption/production schedules (e.g., production values are inverted to be positive). However, the persistence layer was then applying the default power sensor sign logic again, negating this carefully applied inversion. This resulted in production sensors receiving negative values when they should receive positive values, and vice versa for consumption sensors. Solution: Skip the default sign inversion logic for consumption and production output schedules (identified by result["name"] being "consumption_schedule" or "production_schedule"), as their sign convention is already correctly encoded by the scheduler. --- flexmeasures/data/services/scheduling.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 8c32c01c72..86da3ba72a 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -639,8 +639,20 @@ def make_schedule( # noqa: C901 sign = 1 - if result["sensor"].measures_power and not result["sensor"].get_attribute( - "consumption_is_positive", False + # Skip sign logic for consumption/production output schedules, as their sign + # convention is already handled by the scheduler (defined by the field name -- + # "consumption" means consumption positive; "production" means production positive). + # Only apply sign logic to the main power schedules ("storage_schedule", etc). + result_name = result.get("name", "") + is_consumption_production_output = result_name in ( + "consumption_schedule", + "production_schedule", + ) + + if ( + not is_consumption_production_output + and result["sensor"].measures_power + and not result["sensor"].get_attribute("consumption_is_positive", False) ): sign = -1 From 7d902c3e0223a0217b73471c857494d4556e1a7f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 22:01:29 +0200 Subject: [PATCH 22/44] =?UTF-8?q?fix:=20refine=20sign=20inversion=20logic?= =?UTF-8?q?=20for=20consumption/production=20output=20sensors=20The=20prev?= =?UTF-8?q?ious=20fix=20was=20too=20broad=20-=20it=20skipped=20sign=20inve?= =?UTF-8?q?rsion=20for=20any=20result=20named=20"consumption=5Fschedule"?= =?UTF-8?q?=20or=20"production=5Fschedule",=20but=20this=20broke=20backwar?= =?UTF-8?q?ds=20compatibility=20with=20custom=20schedulers=20that=20return?= =?UTF-8?q?=20Series=20wrapped=20as=20such.=20The=20key=20distinction=20is?= =?UTF-8?q?=20whether=20the=20result=20sensor=20is=20the=20main=20asset=5F?= =?UTF-8?q?or=5Fsensor=20(main=20power=20schedule)=20or=20a=20dedicated=20?= =?UTF-8?q?output=20sensor=20(consumption/production=20split).=20New=20log?= =?UTF-8?q?ic:=20-=20If=20result=5Fsensor=20=3D=3D=20asset=5For=5Fsensor?= =?UTF-8?q?=20(or=20its=20sensor):=20it's=20the=20main=20power=20schedule?= =?UTF-8?q?=20=20=20=E2=86=92=20Apply=20default=20sign=20logic=20(backward?= =?UTF-8?q?s=20compat=20and=20regular=20storage=20schedules)=20-=20If=20re?= =?UTF-8?q?sult=5Fsensor=20is=20different=20AND=20result=20name=20is=20con?= =?UTF-8?q?sumption/production:=20it's=20an=20=20=20output=20sensor=20?= =?UTF-8?q?=E2=86=92=20Skip=20sign=20logic=20(already=20handled=20by=20sch?= =?UTF-8?q?eduler)=20This=20ensures=20both=20the=20new=20consumption/produ?= =?UTF-8?q?ction=20output=20features=20and=20existing=20custom=20scheduler?= =?UTF-8?q?s=20work=20correctly.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 86da3ba72a..beed5225b6 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -642,11 +642,18 @@ def make_schedule( # noqa: C901 # Skip sign logic for consumption/production output schedules, as their sign # convention is already handled by the scheduler (defined by the field name -- # "consumption" means consumption positive; "production" means production positive). - # Only apply sign logic to the main power schedules ("storage_schedule", etc). + # These are identified by having a result name of "consumption_schedule" or + # "production_schedule" AND being stored on a sensor that is not the main asset_or_sensor. + # The main power schedules (backwards compat, "storage_schedule", etc) use the main sensor. result_name = result.get("name", "") - is_consumption_production_output = result_name in ( - "consumption_schedule", - "production_schedule", + result_sensor = result["sensor"] + is_main_power_schedule = result_sensor == asset_or_sensor or ( + hasattr(asset_or_sensor, "generic_asset") + and result_sensor.generic_asset == asset_or_sensor + ) + is_consumption_production_output = ( + result_name in ("consumption_schedule", "production_schedule") + and not is_main_power_schedule ) if ( From 82da372496e2ad36724b95efb2f8be3ab8b582c2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 22:16:32 +0200 Subject: [PATCH 23/44] refactor: extract sign resolution logic to utility function New function _resolve_schedule_output_sign() encapsulates the logic for determining whether sign conversion is needed for a schedule output result. Benefits: - Clearer expression of intent (function name documents purpose) - Proper RST docstring with parameter and return descriptions - Inline comments explain the differentiation between main power schedules and consumption/production output schedules - Easier to test and reuse logic if needed elsewhere - make_schedule() is now more readable with the core logic delegated No functional changes - purely a refactoring for code clarity. --- flexmeasures/data/services/scheduling.py | 75 ++++++++++++++++-------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index beed5225b6..74652ee4b5 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -522,6 +522,55 @@ def create_simultaneous_scheduling_job( return job +def _resolve_schedule_output_sign( + result: dict, + asset_or_sensor: Asset | Sensor, +) -> int: + """Determine the sign multiplier for a schedule output result. + + Returns 1 (no sign change) or -1 (invert sign) depending on whether the result + is a power schedule that needs sign conversion to match FlexMeasures convention + (consumption positive, production negative). + + For consumption/production output schedules (identified by result name and sensor), + the sign is already correct per the scheduler, so no conversion is applied. + For other power schedules (main power sensors), the standard conversion is applied. + + :param result: Schedule output result dict with keys 'name', 'sensor', 'data'. + :param asset_or_sensor: The Asset or Sensor being scheduled (main power sensor). + :return: Sign multiplier: 1 (keep sign) or -1 (invert sign). + """ + # Check if this is a consumption/production output schedule (dedicated sensor) + # vs. main power schedule (same as asset_or_sensor). + result_name = result.get("name", "") + result_sensor = result["sensor"] + + # Identify if the result uses the main power sensor + is_main_power_schedule = result_sensor == asset_or_sensor or ( + hasattr(asset_or_sensor, "generic_asset") + and result_sensor.generic_asset == asset_or_sensor + ) + + # Consumption/production output schedules have their sign convention already + # encoded in the field name ("consumption" = consumption positive; + # "production" = production positive). Only main power schedules need conversion. + is_consumption_production_output = ( + result_name in ("consumption_schedule", "production_schedule") + and not is_main_power_schedule + ) + + # Apply standard sign inversion only for main power schedules that measure power + # and don't have the consumption_is_positive attribute. + if ( + not is_consumption_production_output + and result_sensor.measures_power + and not result_sensor.get_attribute("consumption_is_positive", False) + ): + return -1 + + return 1 + + def make_schedule( # noqa: C901 sensor_id: int | None = None, start: datetime | None = None, @@ -637,31 +686,7 @@ def make_schedule( # noqa: C901 if "sensor" not in result: continue - sign = 1 - - # Skip sign logic for consumption/production output schedules, as their sign - # convention is already handled by the scheduler (defined by the field name -- - # "consumption" means consumption positive; "production" means production positive). - # These are identified by having a result name of "consumption_schedule" or - # "production_schedule" AND being stored on a sensor that is not the main asset_or_sensor. - # The main power schedules (backwards compat, "storage_schedule", etc) use the main sensor. - result_name = result.get("name", "") - result_sensor = result["sensor"] - is_main_power_schedule = result_sensor == asset_or_sensor or ( - hasattr(asset_or_sensor, "generic_asset") - and result_sensor.generic_asset == asset_or_sensor - ) - is_consumption_production_output = ( - result_name in ("consumption_schedule", "production_schedule") - and not is_main_power_schedule - ) - - if ( - not is_consumption_production_output - and result["sensor"].measures_power - and not result["sensor"].get_attribute("consumption_is_positive", False) - ): - sign = -1 + sign = _resolve_schedule_output_sign(result, asset_or_sensor) ts_value_schedule = [ TimedBelief( From 40128b73248689d47e9878dc4411160a5c457e01 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 26 May 2026 22:53:22 +0200 Subject: [PATCH 24/44] fix: explicitly set the consumption_is_positive attribute on consumption and production sensors to make sure the get_schedule doesn't flip the sign Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 74652ee4b5..5140739036 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -710,6 +710,23 @@ def make_schedule( # noqa: C901 if not dry_run: save_to_db(bdf) + + # For consumption/production output sensors, explicitly set the consumption_is_positive + # attribute so that future get_schedule calls on these sensors apply the correct sign + # convention. This is necessary because the job has a TTL and won't be available forever. + result_name = result.get("name", "") + result_sensor = result["sensor"] + is_main_power_schedule = result_sensor == asset_or_sensor or ( + hasattr(asset_or_sensor, "generic_asset") + and result_sensor.generic_asset == asset_or_sensor + ) + + if result_name == "consumption_schedule" and not is_main_power_schedule: + # Consumption sensor: consumption is stored as positive + result_sensor.set_attribute("consumption_is_positive", True) + elif result_name == "production_schedule" and not is_main_power_schedule: + # Production sensor: production is stored as positive (consumption negative) + result_sensor.set_attribute("consumption_is_positive", False) else: print( f"\nNot saving schedule for sensor `{bdf.sensor}` to the database (because of dry-run), but this is what I computed:\n{bdf}" From 9e31174d1d2bda0254d405ad3b9fed166d079324 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 27 May 2026 17:22:58 +0200 Subject: [PATCH 25/44] feat: test fetching schedules from consumption and production sensors Signed-off-by: F.N. Claessen --- .../tests/test_sensor_schedules_fresh_db.py | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index b6cc2989dd..17a0409dd7 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -640,6 +640,251 @@ def test_multiple_contracts( ) +@pytest.mark.parametrize( + "output_type", + ["consumption", "production"], +) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_get_schedule_sign_convention_json_flex_model( + app, + fresh_db, + add_market_prices_fresh_db, + add_battery_assets_fresh_db, + battery_soc_sensor_fresh_db, + keep_scheduling_queue_empty, + output_type, + requesting_user, +): + """Test that get_schedule returns correct sign conventions for consumption/production + output sensors when the flex-model is passed via JSON body. + + Consumption output sensor: consumption positive, production negative. + Production output sensor: production positive, consumption negative. + """ + battery_asset = add_battery_assets_fresh_db["Test battery"] + + # Create a dedicated output sensor on the battery asset + output_sensor = Sensor( + name=f"{output_type} output", + generic_asset=battery_asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + fresh_db.session.add(output_sensor) + fresh_db.session.flush() + + # Build the trigger message with the output sensor in the flex-model + message = message_for_trigger_schedule(resolution="PT1H") + price_sensor_id = add_market_prices_fresh_db["epex_da"].id + message["flex-context"] = { + "consumption-price": {"sensor": price_sensor_id}, + "production-price": {"sensor": price_sensor_id}, + "site-power-capacity": "1 TW", + } + message["flex-model"][output_type] = {"sensor": output_sensor.id} + + # Find the main power sensor to trigger the schedule on + sensor = ( + Sensor.query.filter(Sensor.name == "power") + .join(GenericAsset, GenericAsset.id == Sensor.generic_asset_id) + .filter(GenericAsset.name == "Test battery") + .one_or_none() + ) + + with app.test_client() as client: + trigger_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + assert trigger_response.status_code == 200 + job_id = trigger_response.json["schedule"] + + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) + assert job.is_finished + + # Retrieve the schedule from the output sensor + with app.test_client() as client: + get_response = client.get( + url_for("SensorAPI:get_schedule", id=output_sensor.id, uuid=job_id), + query_string={"duration": "PT24H"}, + ) + assert get_response.status_code == 200 + values = get_response.json["values"] + + # The battery schedule should have non-zero values (charging and/or discharging) + assert len(values) > 0 + assert any(v != 0 for v in values), "Schedule should have non-zero values" + + # Get the main power sensor schedule for comparison (consumption positive convention) + with app.test_client() as client: + main_response = client.get( + url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), + query_string={"duration": "PT24H"}, + ) + assert main_response.status_code == 200 + main_values = main_response.json["values"] + + if output_type == "consumption": + # Consumption sensor returns the full schedule in consumption-positive convention + # (same sign as main power sensor schedule) + # Both should have the same absolute pattern, consumption positive + for main_val, out_val in zip(main_values, values): + if main_val > 0: + # Charging (consumption) should be positive on consumption sensor + assert ( + out_val >= 0 + ), f"Consumption sensor should show positive for charging: main={main_val}, output={out_val}" + elif main_val < 0: + # Discharging (production) should be negative on consumption sensor + assert ( + out_val <= 0 + ), f"Consumption sensor should show negative for discharging: main={main_val}, output={out_val}" + elif output_type == "production": + # Production sensor returns the full schedule in production-positive convention + # (inverted sign relative to main power sensor schedule) + for main_val, out_val in zip(main_values, values): + if main_val < 0: + # Discharging (production) should be positive on production sensor + assert ( + out_val >= 0 + ), f"Production sensor should show positive for discharging: main={main_val}, output={out_val}" + elif main_val > 0: + # Charging (consumption) should be negative on production sensor + assert ( + out_val <= 0 + ), f"Production sensor should show negative for charging: main={main_val}, output={out_val}" + + +@pytest.mark.parametrize( + "output_type", + ["consumption", "production"], +) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_get_schedule_sign_convention_db_flex_model( + app, + fresh_db, + add_market_prices_fresh_db, + add_battery_assets_fresh_db, + battery_soc_sensor_fresh_db, + keep_scheduling_queue_empty, + output_type, + requesting_user, +): + """Test that get_schedule returns correct sign conventions for consumption/production + output sensors when the flex-model is stored in the DB (asset attributes). + + Consumption output sensor: consumption positive, production negative. + Production output sensor: production positive, consumption negative. + """ + battery_asset = add_battery_assets_fresh_db["Test battery"] + + # Create a dedicated output sensor on the battery asset + output_sensor = Sensor( + name=f"{output_type} output db", + generic_asset=battery_asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + fresh_db.session.add(output_sensor) + fresh_db.session.flush() + + # Store the output sensor reference in the asset's flex_model (DB flex-model) + battery_asset.flex_model = battery_asset.flex_model or {} + battery_asset.flex_model[output_type] = {"sensor": output_sensor.id} + fresh_db.session.add(battery_asset) + fresh_db.session.flush() + + # Build the trigger message without consumption/production in JSON flex-model + # (it will be read from the asset's DB flex-model) + message = message_for_trigger_schedule(resolution="PT1H") + price_sensor_id = add_market_prices_fresh_db["epex_da"].id + message["flex-context"] = { + "consumption-price": {"sensor": price_sensor_id}, + "production-price": {"sensor": price_sensor_id}, + "site-power-capacity": "1 TW", + } + + # Find the main power sensor to trigger the schedule on + sensor = ( + Sensor.query.filter(Sensor.name == "power") + .join(GenericAsset, GenericAsset.id == Sensor.generic_asset_id) + .filter(GenericAsset.name == "Test battery") + .one_or_none() + ) + + # Flush Redis to clear any job dedup cache from prior parametrized runs + app.redis_connection.flushdb() + + with app.test_client() as client: + trigger_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + assert trigger_response.status_code == 200 + job_id = trigger_response.json["schedule"] + + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) + assert job.is_finished + + # Retrieve the schedule from the output sensor + with app.test_client() as client: + get_response = client.get( + url_for("SensorAPI:get_schedule", id=output_sensor.id, uuid=job_id), + query_string={"duration": "PT24H"}, + ) + assert ( + get_response.status_code == 200 + ), f"GET schedule failed: {get_response.json}" + values = get_response.json["values"] + + # The battery schedule should have non-zero values + assert len(values) > 0 + assert any(v != 0 for v in values), "Schedule should have non-zero values" + + # Get the main power sensor schedule for comparison + with app.test_client() as client: + main_response = client.get( + url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), + query_string={"duration": "PT24H"}, + ) + assert main_response.status_code == 200 + main_values = main_response.json["values"] + + if output_type == "consumption": + for main_val, out_val in zip(main_values, values): + if main_val > 0: + assert ( + out_val >= 0 + ), f"Consumption sensor should show positive for charging: main={main_val}, output={out_val}" + elif main_val < 0: + assert ( + out_val <= 0 + ), f"Consumption sensor should show negative for discharging: main={main_val}, output={out_val}" + elif output_type == "production": + for main_val, out_val in zip(main_values, values): + if main_val < 0: + assert ( + out_val >= 0 + ), f"Production sensor should show positive for discharging: main={main_val}, output={out_val}" + elif main_val > 0: + assert ( + out_val <= 0 + ), f"Production sensor should show negative for charging: main={main_val}, output={out_val}" + + # Clean up: remove the flex_model entry so it doesn't affect other tests + del battery_asset.flex_model[output_type] + fresh_db.session.add(battery_asset) + fresh_db.session.flush() + + def setup_inflexible_device_sensors(fresh_db, asset, sensor_name, sensor_num): """Test helper function to add sensor_num sensors to an asset""" sensors = list() From 6eae5c81cc1d7a1289270a415b708355af965752 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 27 May 2026 17:28:53 +0200 Subject: [PATCH 26/44] fix: set_attribute is a no-op when the attribute does not exist; also refactor Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 72 ++++++++++++++---------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 5140739036..9f825d7923 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -522,6 +522,34 @@ def create_simultaneous_scheduling_job( return job +def _is_consumption_production_output( + result: dict, asset_or_sensor: Asset | Sensor +) -> bool: + """Return True when *result* is a dedicated consumption or production output schedule. + + A dedicated output schedule is one whose sensor is different from the main asset_or_sensor being scheduled, + and whose name is ``"consumption_schedule"`` or ``"production_schedule"``. + The main power schedule (including the backwards-compat wrapper for custom schedulers that return a plain Series) + uses the same sensor as *asset_or_sensor* and is therefore not considered an output schedule. + + :param result: Schedule output result dict with keys 'name', 'sensor', 'data'. + :param asset_or_sensor: The Asset or Sensor being scheduled (main power sensor). + :return: True when the result targets a dedicated output sensor. + """ + result_name = result.get("name", "") + if result_name not in ("consumption_schedule", "production_schedule"): + return False + + result_sensor = result["sensor"] + # The main power schedule uses the same object as asset_or_sensor, + # or – when asset_or_sensor is an Asset – one of its sensors. + is_main = result_sensor == asset_or_sensor or ( + hasattr(asset_or_sensor, "generic_asset") + and result_sensor.generic_asset == asset_or_sensor + ) + return not is_main + + def _resolve_schedule_output_sign( result: dict, asset_or_sensor: Asset | Sensor, @@ -540,31 +568,18 @@ def _resolve_schedule_output_sign( :param asset_or_sensor: The Asset or Sensor being scheduled (main power sensor). :return: Sign multiplier: 1 (keep sign) or -1 (invert sign). """ - # Check if this is a consumption/production output schedule (dedicated sensor) - # vs. main power schedule (same as asset_or_sensor). - result_name = result.get("name", "") result_sensor = result["sensor"] - # Identify if the result uses the main power sensor - is_main_power_schedule = result_sensor == asset_or_sensor or ( - hasattr(asset_or_sensor, "generic_asset") - and result_sensor.generic_asset == asset_or_sensor - ) - # Consumption/production output schedules have their sign convention already # encoded in the field name ("consumption" = consumption positive; # "production" = production positive). Only main power schedules need conversion. - is_consumption_production_output = ( - result_name in ("consumption_schedule", "production_schedule") - and not is_main_power_schedule - ) + if _is_consumption_production_output(result, asset_or_sensor): + return 1 # Apply standard sign inversion only for main power schedules that measure power # and don't have the consumption_is_positive attribute. - if ( - not is_consumption_production_output - and result_sensor.measures_power - and not result_sensor.get_attribute("consumption_is_positive", False) + if result_sensor.measures_power and not result_sensor.get_attribute( + "consumption_is_positive", False ): return -1 @@ -714,19 +729,16 @@ def make_schedule( # noqa: C901 # For consumption/production output sensors, explicitly set the consumption_is_positive # attribute so that future get_schedule calls on these sensors apply the correct sign # convention. This is necessary because the job has a TTL and won't be available forever. - result_name = result.get("name", "") - result_sensor = result["sensor"] - is_main_power_schedule = result_sensor == asset_or_sensor or ( - hasattr(asset_or_sensor, "generic_asset") - and result_sensor.generic_asset == asset_or_sensor - ) - - if result_name == "consumption_schedule" and not is_main_power_schedule: - # Consumption sensor: consumption is stored as positive - result_sensor.set_attribute("consumption_is_positive", True) - elif result_name == "production_schedule" and not is_main_power_schedule: - # Production sensor: production is stored as positive (consumption negative) - result_sensor.set_attribute("consumption_is_positive", False) + if _is_consumption_production_output(result, asset_or_sensor): + result_sensor = result["sensor"] + result_name = result.get("name", "") + # Consumption sensor stores consumption as positive; production sensor stores + # production as positive (i.e. consumption_is_positive is False). + intended = result_name == "consumption_schedule" + # Direct attribute assignment works for both new and existing attributes. + # set_attribute() is intentionally not used here because it silently + # no-ops when the attribute does not yet exist. + result_sensor.attributes["consumption_is_positive"] = intended else: print( f"\nNot saving schedule for sensor `{bdf.sensor}` to the database (because of dry-run), but this is what I computed:\n{bdf}" From 2cd8d2571a8f3b50dc33cc6620fe51c7e6636dcd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 27 May 2026 17:29:12 +0200 Subject: [PATCH 27/44] feat: prevent mutating consumption_is_positive once it is set Signed-off-by: F.N. Claessen --- flexmeasures/data/services/scheduling.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 9f825d7923..403ddb9a05 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -735,6 +735,14 @@ def make_schedule( # noqa: C901 # Consumption sensor stores consumption as positive; production sensor stores # production as positive (i.e. consumption_is_positive is False). intended = result_name == "consumption_schedule" + existing = result_sensor.attributes.get("consumption_is_positive") + if existing is not None and existing != intended: + raise ValueError( + f"Sensor {result_sensor} already has `consumption_is_positive={existing}`, " + f"which conflicts with the '{result_name}' output schedule " + f"(expected `consumption_is_positive={intended}`). " + f"Remove or correct the attribute before re-running the scheduler." + ) # Direct attribute assignment works for both new and existing attributes. # set_attribute() is intentionally not used here because it silently # no-ops when the attribute does not yet exist. From e2ccc08969268f4eaf5b82cc791d96ebcefeafca Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 27 May 2026 17:52:06 +0200 Subject: [PATCH 28/44] data/services/scheduling: extract _set_output_sensor_consumption_is_positive util Context: - The inline attribute-update block in make_schedule was not encapsulated, making it harder to read and reuse. - The conflict check happened after save_to_db, so a mismatched sensor could silently persist some beliefs before raising. Change: - Extract attribute check and assignment into _set_output_sensor_consumption_is_positive(). - Call it before save_to_db so the ValueError is raised before any data is written (fail-fast). --- flexmeasures/data/services/scheduling.py | 69 +++++++++++++++-------- flexmeasures/ui/static/openapi-specs.json | 6 +- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 403ddb9a05..561d67a1a8 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -550,6 +550,50 @@ def _is_consumption_production_output( return not is_main +def _set_output_sensor_consumption_is_positive( + result: dict, asset_or_sensor: Asset | Sensor +) -> None: + """Set the ``consumption_is_positive`` attribute on a dedicated output sensor. + + For consumption output sensors the attribute is set to ``True`` (consumption is stored as + positive values). For production output sensors it is set to ``False`` (production is stored + as positive values, consumption as negative). + + The function is a no-op when *result* is not a dedicated consumption/production output + schedule (as determined by :func:`_is_consumption_production_output`). + + A ``ValueError`` is raised when the attribute is already present on the sensor but points + in the wrong direction for the flex-model field being used. This check runs *before* any + data are written so that the error surfaces as early as possible. + + :param result: Schedule output result dict with keys ``'name'``, ``'sensor'``, + ``'data'``. + :param asset_or_sensor: The Asset or Sensor being scheduled (main power sensor). + :raises ValueError: When ``consumption_is_positive`` is already set to the wrong value + for the given flex-model field. + """ + if not _is_consumption_production_output(result, asset_or_sensor): + return + + result_sensor = result["sensor"] + result_name = result.get("name", "") + # consumption_schedule → True (consumption positive) + # production_schedule → False (production positive, i.e. consumption negative) + intended = result_name == "consumption_schedule" + existing = result_sensor.attributes.get("consumption_is_positive") + if existing is not None and existing != intended: + raise ValueError( + f"Sensor {result_sensor} already has `consumption_is_positive={existing}`, " + f"which conflicts with the '{result_name}' output schedule " + f"(expected `consumption_is_positive={intended}`). " + f"Remove or correct the attribute before re-running the scheduler." + ) + # Direct attribute assignment works for both new and existing attributes. + # set_attribute() is intentionally not used here because it silently + # no-ops when the attribute does not yet exist. + result_sensor.attributes["consumption_is_positive"] = intended + + def _resolve_schedule_output_sign( result: dict, asset_or_sensor: Asset | Sensor, @@ -724,29 +768,10 @@ def make_schedule( # noqa: C901 bdf = bdf.resample_events(bdf.sensor.event_resolution) if not dry_run: + # Validate and set the consumption_is_positive attribute before writing data so + # that a conflict raises an error early, before any beliefs are persisted. + _set_output_sensor_consumption_is_positive(result, asset_or_sensor) save_to_db(bdf) - - # For consumption/production output sensors, explicitly set the consumption_is_positive - # attribute so that future get_schedule calls on these sensors apply the correct sign - # convention. This is necessary because the job has a TTL and won't be available forever. - if _is_consumption_production_output(result, asset_or_sensor): - result_sensor = result["sensor"] - result_name = result.get("name", "") - # Consumption sensor stores consumption as positive; production sensor stores - # production as positive (i.e. consumption_is_positive is False). - intended = result_name == "consumption_schedule" - existing = result_sensor.attributes.get("consumption_is_positive") - if existing is not None and existing != intended: - raise ValueError( - f"Sensor {result_sensor} already has `consumption_is_positive={existing}`, " - f"which conflicts with the '{result_name}' output schedule " - f"(expected `consumption_is_positive={intended}`). " - f"Remove or correct the attribute before re-running the scheduler." - ) - # Direct attribute assignment works for both new and existing attributes. - # set_attribute() is intentionally not used here because it silently - # no-ops when the attribute does not yet exist. - result_sensor.attributes["consumption_is_positive"] = intended else: print( f"\nNot saving schedule for sensor `{bdf.sensor}` to the database (because of dry-run), but this is what I computed:\n{bdf}" diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index d249094501..3d5d01c74d 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -6000,14 +6000,14 @@ "type": "object", "properties": { "consumption": { - "description": "Sensor used to record the scheduled consumption.", + "description": "Sensor used to record the scheduled power as seen from a consumption perspective.\n\nThe sign convention is determined by the key name, so the sensor itself does not need a consumption_is_positive attribute.\n\nDepending on which output sensors are defined:\n\n- Only consumption defined: the full power schedule is stored on this sensor using the\n standard FlexMeasures sign convention (consumption positive, production negative).\n- Only production defined: the full power schedule is stored on the production sensor\n with the sign inverted (production positive, consumption negative).\n- Both defined: only the non-negative part of the schedule is stored on this sensor (zero for\n time steps with net production), and only the non-positive part (sign-flipped) is stored on the\n production sensor.\n", "example": { "sensor": 14 }, "$ref": "#/components/schemas/SensorReference" }, "production": { - "description": "Sensor used to record the scheduled production.", + "description": "Sensor used to record the scheduled power as seen from a production perspective.\n\nThe sign convention is determined by the key name, so the sensor itself does not need a consumption_is_positive attribute.\n\nSee consumption for the full description of the split logic when both sensors are defined.\n", "example": { "sensor": 15 }, @@ -6128,7 +6128,7 @@ "example": "90%" }, "storage-efficiency": { - "description": "The efficiency of keeping the storage's state of charge at its present level, used to encode losses over time.\nAs a result, each time step the energy is held longer leads to higher losses.\nThis setting is crucial to some sorts of energy storage, e.g. thermal buffers.\nTo give an example, when this setting is at 95% (or 0.95), this means a loss of 5% per time step. Defaults to 100% (no storage loss over time).\nNote that the storage efficiency used by the scheduler is applied over each time step equal to the sensor resolution.\nFor example, a storage efficiency of 95 percent per (absolute) day, for scheduling a 1-hour resolution sensor, should be passed as a storage efficiency of 0.951/24 = 0.997865.\n", + "description": "The efficiency of keeping the storage's state of charge at its present level, used to encode losses over time.\nAs a result, each time step the energy is held longer leads to higher losses.\nThis setting is crucial to some sorts of energy storage, e.g. thermal buffers.\nTo give an example, when this setting is at 95% (or 0.95), this means a loss of 5% per time step. Defaults to 100% (no storage loss over time).\nNote that the storage efficiency used by the scheduler is applied over each time step equal to the scheduling resolution.\nFor example, a storage efficiency of 95 percent per (absolute) day, for scheduling a 1-hour resolution sensor, should be passed as a storage efficiency of 0.951/24 = 0.997865.\nAlternatively, to let FlexMeasures handle the conversion for you, record the storage-efficiency on a dedicated sensor (in this example, with a 24-hour event resolution).\nThen reference that sensor in the storage-efficiency field.\n", "example": "99.9%", "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, From e555ea47309ea6f92bc68eea285e29ddc048b11a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 27 May 2026 17:52:17 +0200 Subject: [PATCH 29/44] api/v3_0/tests: add test for conflicting consumption_is_positive attribute Context: - Verify that make_schedule raises ValueError and writes no data when a sensor pre-marked as consumption-only is referenced under the production flex-model field. Change: - Add test_conflicting_consumption_is_positive_attribute_prevents_data_save. - Use force-new-job-creation in the trigger request to bypass the job-cache, which would otherwise return the cached result of a prior test that used the same sensor ID and time window (fresh_db resets IDs per function, producing the same cache key). --- .../tests/test_sensor_schedules_fresh_db.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index 17a0409dd7..008b602282 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -299,6 +299,99 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( ), "expected first hour to show positive production" +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_conflicting_consumption_is_positive_attribute_prevents_data_save( + app, + fresh_db, + add_market_prices_fresh_db, + add_battery_assets_fresh_db, + battery_soc_sensor_fresh_db, + setup_sources_fresh_db, + keep_scheduling_queue_empty, + requesting_user, +): + """Scheduling must fail before persisting any data when consumption_is_positive is already + set to a value that conflicts with the flex-model field used for the sensor. + + Here a sensor that already has ``consumption_is_positive=True`` (consumption sensor) is + mistakenly referenced under the ``production`` flex-model field. The scheduler should raise a + ``ValueError`` and leave the sensor's belief table empty. + + ``force-new-job-creation`` is set to bypass the job-cache, which would otherwise serve the + cached result of a prior test that ran against the same battery sensor and time window + (``fresh_db`` resets sensor IDs to the same values each test, producing the same cache key). + """ + message = message_for_trigger_schedule(resolution="PT1H") + message["force-new-job-creation"] = True + message["flex-context"] = { + "consumption-price": {"sensor": add_market_prices_fresh_db["epex_da"].id}, + "production-price": {"sensor": add_market_prices_fresh_db["epex_da"].id}, + "site-power-capacity": "1 TW", + } + message["flex-model"].pop("soc-at-start") + message["flex-model"]["state-of-charge"] = { + "sensor": battery_soc_sensor_fresh_db.id + } + + # Find the main power sensor for the battery. + power_sensor = ( + Sensor.query.filter(Sensor.name == "power") + .join(GenericAsset, GenericAsset.id == Sensor.generic_asset_id) + .filter(GenericAsset.name == "Test battery") + .one_or_none() + ) + + # Create an output sensor that is pre-marked as a *consumption* sensor. + # We will then (wrongly) pass it as the `production` field in the flex-model. + mismatched_sensor = Sensor( + name="consumption output (misused as production)", + generic_asset=power_sensor.generic_asset, + unit="MW", + event_resolution=power_sensor.event_resolution, + attributes={"consumption_is_positive": True}, # consumption sensor + ) + fresh_db.session.add(mismatched_sensor) + fresh_db.session.add( + TimedBelief( + sensor=battery_soc_sensor_fresh_db, + source=setup_sources_fresh_db["Seita"], + event_start=parse_datetime(message["start"]), + belief_horizon=timedelta(0), + event_value=50, + ) + ) + fresh_db.session.commit() + + # Reference the consumption-marked sensor under the `production` field — a mismatch. + message["flex-model"]["production"] = {"sensor": mismatched_sensor.id} + + with app.test_client() as client: + trigger_response = client.post( + url_for("SensorAPI:trigger_schedule", id=power_sensor.id), + json=message, + ) + assert trigger_response.status_code == 200 + job_id = trigger_response.json["schedule"] + + work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) + + # The job should have failed due to the attribute conflict. + job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) + assert job.is_failed, "Expected the scheduling job to fail on attribute conflict" + assert "consumption_is_positive" in str(job.meta.get("exception", "")) + + # No beliefs should have been written for the mismatched sensor. + mismatched_sensor = fresh_db.session.get(Sensor, mismatched_sensor.id) + saved_beliefs = mismatched_sensor.search_beliefs( + event_starts_after=parse_datetime(message["start"]) + ) + assert ( + len(saved_beliefs) == 0 + ), "No data should be saved when the attribute conflict is detected before save_to_db" + + @pytest.mark.parametrize( "context_sensor, asset_sensor, parent_sensor, expect_sensor", [ From 7038e53f4047e9c44de7da65e702413dc6a22acb Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 10:05:35 +0200 Subject: [PATCH 30/44] fix: test expectations for production sensors Signed-off-by: F.N. Claessen --- .../tests/test_sensor_schedules_fresh_db.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index 008b602282..50101ca013 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -754,7 +754,7 @@ def test_get_schedule_sign_convention_json_flex_model( output sensors when the flex-model is passed via JSON body. Consumption output sensor: consumption positive, production negative. - Production output sensor: production positive, consumption negative. + Production output sensor: consumption positive, production negative. """ battery_asset = add_battery_assets_fresh_db["Test battery"] @@ -837,19 +837,19 @@ def test_get_schedule_sign_convention_json_flex_model( out_val <= 0 ), f"Consumption sensor should show negative for discharging: main={main_val}, output={out_val}" elif output_type == "production": - # Production sensor returns the full schedule in production-positive convention - # (inverted sign relative to main power sensor schedule) + # Production sensor returns the full schedule in consumption-positive convention + # (same sign as main power sensor schedule) for main_val, out_val in zip(main_values, values): - if main_val < 0: - # Discharging (production) should be positive on production sensor + if main_val > 0: + # Charging (consumption) should be positive on production sensor assert ( out_val >= 0 - ), f"Production sensor should show positive for discharging: main={main_val}, output={out_val}" - elif main_val > 0: - # Charging (consumption) should be negative on production sensor + ), f"Production sensor should show positive for charging: main={main_val}, output={out_val}" + elif main_val < 0: + # Discharging (production) should be negative on production sensor assert ( out_val <= 0 - ), f"Production sensor should show negative for charging: main={main_val}, output={out_val}" + ), f"Production sensor should show negative for discharging: main={main_val}, output={out_val}" @pytest.mark.parametrize( @@ -963,14 +963,14 @@ def test_get_schedule_sign_convention_db_flex_model( ), f"Consumption sensor should show negative for discharging: main={main_val}, output={out_val}" elif output_type == "production": for main_val, out_val in zip(main_values, values): - if main_val < 0: + if main_val > 0: assert ( out_val >= 0 - ), f"Production sensor should show positive for discharging: main={main_val}, output={out_val}" - elif main_val > 0: + ), f"Production sensor should show positive for charging: main={main_val}, output={out_val}" + elif main_val < 0: assert ( out_val <= 0 - ), f"Production sensor should show negative for charging: main={main_val}, output={out_val}" + ), f"Production sensor should show negative for discharging: main={main_val}, output={out_val}" # Clean up: remove the flex_model entry so it doesn't affect other tests del battery_asset.flex_model[output_type] From a2a2b950f71b81fe29426d5484d7b262e5f32f65 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 10:05:57 +0200 Subject: [PATCH 31/44] docs: clarify sign convention for the get_schedule endpoint Signed-off-by: F.N. Claessen --- documentation/api/notation.rst | 2 ++ documentation/concepts/data-model.rst | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index 75a9e0eb26..f814722710 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -302,3 +302,5 @@ We'd recommend to use positive power values to indicate consumption and negative Read more at :ref:`signs_of_power_beliefs` about our treatment of data, which includes data you send in, or you get from forecasts and schedules (hint: you are free to define the sign for your data, but it might affect how you receive your schedules). + +Note that the ``GET /api/v3_0/sensors//schedules/`` endpoint always returns schedules with consumption as positive values and production as negative values. diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index 6c2267768a..c1734cf831 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -170,7 +170,9 @@ We assume that this is what users send in. Note that, if forecasts are created, they will have the same sign as original data. -For schedules, the sign of resulting power data (beliefs) is being switched when data is stored (assuming consumption , and you can prevent that by setting ``sensor.attributes["consumption_is_positive"] = True``. +For schedules, the sign of resulting power data (beliefs) is being switched when data is stored (assuming production is positive), and you can prevent that by setting ``sensor.attributes["consumption_is_positive"] = True``. +On sensors that have been referenced in a flex-model under the ``consumption`` or ``production`` field, this attribute is automatically set. +The ``GET /api/v3_0/sensors//schedules/`` endpoint always returns schedules with consumption as positive values and production as negative values. .. note:: We will soon document better what the scheduler does in detail, and how the attribute works. From dfbc2a0bce2e411b5022c46818c313cc053fed79 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 10:22:58 +0200 Subject: [PATCH 32/44] feat: add sign-convention query parameter to get_schedule endpoint Context: - The endpoint always returned consumption-positive values, giving callers no way to request production-positive or raw database values. Change: - Add ScheduleSignConvention constants class to scheduling schemas. - Extend GetScheduleSchema with a sign-convention field (default: consumption-positive) validated against the three allowed modes. - Update get_schedule to apply the chosen convention: * consumption-positive: invert DB values when consumption_is_positive=False * production-positive: invert DB values when consumption_is_positive=True * wysiwyg: return raw database values unchanged - Regenerate openapi-specs.json with the new parameter. Signed-off-by: F.N. Claessen feat: add sign-convention query parameter to get_schedule endpoint Context: - The endpoint always returned consumption-positive values, giving callers no way to request production-positive or raw database values. Change: - Add ScheduleSignConvention constants class to scheduling schemas. - Extend GetScheduleSchema with a sign-convention field (default: consumption-positive) validated against the three allowed modes. - Update get_schedule to apply the chosen convention: * consumption-positive: invert DB values when consumption_is_positive=False * production-positive: invert DB values when consumption_is_positive=True * wysiwyg: return raw database values unchanged - Regenerate openapi-specs.json with the new parameter. --- flexmeasures/api/v3_0/sensors.py | 61 ++++++++++++++++--- .../data/schemas/scheduling/__init__.py | 38 ++++++++++++ flexmeasures/ui/static/openapi-specs.json | 17 +++++- 3 files changed, 108 insertions(+), 8 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 615b68d9b3..af33f76cc0 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -67,7 +67,10 @@ ) from flexmeasures.data.schemas import AssetIdField, SourceIdField from flexmeasures.api.common.schemas.search import SearchFilterField -from flexmeasures.data.schemas.scheduling import GetScheduleSchema +from flexmeasures.data.schemas.scheduling import ( + GetScheduleSchema, + ScheduleSignConvention, +) from flexmeasures.data.schemas.units import UnitField from flexmeasures.data.services.sensors import get_sensor_stats from flexmeasures.data.services.sensors import delete_sensor as delete_sensor_and_data @@ -1042,6 +1045,7 @@ def get_schedule( # noqa: C901 job_id: str, duration: timedelta, unit: str | None = None, + sign_convention: str = ScheduleSignConvention.CONSUMPTION_POSITIVE, **kwargs, ): """ @@ -1056,6 +1060,21 @@ def get_schedule( # noqa: C901 - "duration" (6 hours by default; can be increased to plan further into the future) - "unit" (by default, the unit of the schedule is the sensor's unit; a compatible unit can be requested) + - "sign-convention" (controls how power values are signed in the response; see below) + + **Sign convention** + + By default (``sign-convention: consumption-positive``), the endpoint always returns schedules where + consumption is positive and production is negative, regardless of how the values are stored in the + database. This is the most common convention and matches the perspective of a consumer. + + Set ``sign-convention: production-positive`` to flip the sign so that production is returned as + positive and consumption as negative. This matches the perspective of a producer. + + Set ``sign-convention: wysiwyg`` (*what-you-see-is-what-you-get*) to return the raw database values + without any sign adjustment. The values will reflect exactly what is stored, which is determined + by the sensor's ``consumption_is_positive`` attribute (if set) or by the scheduler's default + storage convention (production positive in the database). security: - ApiKeyAuth: [] parameters: @@ -1097,6 +1116,21 @@ def get_schedule( # noqa: C901 example: kW schema: type: string + - in: query + name: sign-convention + required: false + description: | + Sign convention applied to power values in the response. + - ``consumption-positive`` (default): consumption is positive, production is negative. + - ``production-positive``: production is positive, consumption is negative. + - ``wysiwyg``: raw database values returned without sign adjustment. + example: consumption-positive + schema: + type: string + enum: + - consumption-positive + - production-positive + - wysiwyg responses: 200: description: PROCESSED @@ -1223,12 +1257,25 @@ def get_schedule( # noqa: C901 ) sign = 1 - if sensor.measures_power and not sensor.get_attribute( - "consumption_is_positive", False - ): - sign = -1 - - # For consumption schedules, positive values denote consumption. For the db, consumption is negative unless specified explicitly + if sign_convention == ScheduleSignConvention.WYSIWYG: + # Return raw database values without sign adjustment. + # No sign inversion: what's in the DB is what the caller receives. + pass + elif sensor.measures_power: + # Determine whether the database stores consumption as positive or negative. + db_consumption_is_positive = sensor.get_attribute( + "consumption_is_positive", False + ) + if sign_convention == ScheduleSignConvention.CONSUMPTION_POSITIVE: + # Caller wants consumption positive. Invert if DB stores it as negative. + if not db_consumption_is_positive: + sign = -1 + elif sign_convention == ScheduleSignConvention.PRODUCTION_POSITIVE: + # Caller wants production positive. Invert if DB already stores consumption positive. + if db_consumption_is_positive: + sign = -1 + + # Apply sign to get the values in the requested convention consumption_schedule = sign * simplify_index(power_values)["event_value"] if consumption_schedule.empty: # for not in-built schedulers, we are not sure if they would store time series in the db diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 09d5114783..e99b2731d2 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1020,11 +1020,49 @@ def check_flex_model_sensors(self, data, **kwargs): return data +class ScheduleSignConvention: + """Named constants for the three sign-convention modes of the get_schedule endpoint. + + :cvar CONSUMPTION_POSITIVE: Always return schedules with consumption as positive values + and production as negative values. This is the default and + matches the view a *consumer* has of their device. + :cvar PRODUCTION_POSITIVE: Always return schedules with production as positive values + and consumption as negative values. This matches the view a + *producer* (or generator) has of their device. + :cvar WYSIWYG: Return the raw values from the database without any sign inversion, + regardless of the sensor's ``consumption_is_positive`` attribute. + Useful when you want to see exactly what was stored. + """ + + CONSUMPTION_POSITIVE = "consumption-positive" + PRODUCTION_POSITIVE = "production-positive" + WYSIWYG = "wysiwyg" + + ALL = (CONSUMPTION_POSITIVE, PRODUCTION_POSITIVE, WYSIWYG) + + class GetScheduleSchema(Schema): sensor = SensorIdField(required=True, data_key="id") job_id = fields.Str(required=True, data_key="uuid") duration = DurationField(load_default=timedelta(hours=6)) unit = UnitField(load_default=None) + sign_convention = fields.Str( + data_key="sign-convention", + load_default=ScheduleSignConvention.CONSUMPTION_POSITIVE, + validate=validate.OneOf(ScheduleSignConvention.ALL), + metadata=dict( + description=( + "Controls the sign convention applied to schedule values in the response. " + f"``{ScheduleSignConvention.CONSUMPTION_POSITIVE}`` (default): consumption is always returned as positive values " + f"and production as negative values. " + f"``{ScheduleSignConvention.PRODUCTION_POSITIVE}``: production is always returned as positive values " + f"and consumption as negative values. " + f"``{ScheduleSignConvention.WYSIWYG}``: the raw database values are returned without " + "any sign adjustment, regardless of the sensor's ``consumption_is_positive`` attribute." + ), + example=ScheduleSignConvention.CONSUMPTION_POSITIVE, + ), + ) @post_load def finalize_unit_and_duration(self, data, **kwargs): diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 3d5d01c74d..40188b6480 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -690,7 +690,7 @@ "/api/v3_0/sensors/{id}/schedules/{uuid}": { "get": { "summary": "Get schedule for one device", - "description": "Get a schedule from FlexMeasures.\n\nOptional fields:\n\n- \"duration\" (6 hours by default; can be increased to plan further into the future)\n- \"unit\" (by default, the unit of the schedule is the sensor's unit; a compatible unit can be requested)\n", + "description": "Get a schedule from FlexMeasures.\n\nOptional fields:\n\n- \"duration\" (6 hours by default; can be increased to plan further into the future)\n- \"unit\" (by default, the unit of the schedule is the sensor's unit; a compatible unit can be requested)\n- \"sign-convention\" (controls how power values are signed in the response; see below)\n\nSign convention\n\nBy default (sign-convention: consumption-positive), the endpoint always returns schedules where\nconsumption is positive and production is negative, regardless of how the values are stored in the\ndatabase. This is the most common convention and matches the perspective of a consumer.\n\nSet sign-convention: production-positive to flip the sign so that production is returned as\npositive and consumption as negative. This matches the perspective of a producer.\n\nSet sign-convention: wysiwyg (what-you-see-is-what-you-get) to return the raw database values\nwithout any sign adjustment. The values will reflect exactly what is stored, which is determined\nby the sensor's consumption_is_positive attribute (if set) or by the scheduler's default\nstorage convention (production positive in the database).\n", "security": [ { "ApiKeyAuth": [] @@ -736,6 +736,21 @@ "schema": { "type": "string" } + }, + { + "in": "query", + "name": "sign-convention", + "required": false, + "description": "Sign convention applied to power values in the response.\n- consumption-positive (default): consumption is positive, production is negative.\n- production-positive: production is positive, consumption is negative.\n- wysiwyg: raw database values returned without sign adjustment.\n", + "example": "consumption-positive", + "schema": { + "type": "string", + "enum": [ + "consumption-positive", + "production-positive", + "wysiwyg" + ] + } } ], "responses": { From f19088c081a5e74193d1ca16cccf76edb657b42a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 10:23:09 +0200 Subject: [PATCH 33/44] docs: document sign-convention parameter for get_schedule endpoint Context: - The new sign-convention query parameter needs user-facing documentation. Change: - notation.rst: replace single-sentence note with a bulleted list describing all three modes (consumption-positive, production-positive, wysiwyg). - data-model.rst: expand the signs_of_power_beliefs section similarly. - api/change_log.rst: add entry for the new parameter in v3.0-31. --- documentation/api/change_log.rst | 1 + documentation/api/notation.rst | 7 ++++++- documentation/concepts/data-model.rst | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index b11ff609d8..c55d79f149 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -12,6 +12,7 @@ v3.0-31 | 2026-05-20 - Switched from ``force_new_job_creation`` to ``force-new-job-creation`` (maintaining backwards compatibility) and added the field to `/assets/(id)/schedules/trigger` (POST) endpoint, too. - Support both snake_case and kebab-case fields in `/sensors//data` (GET), while only documenting the kebab-case ones. - The ``consumption`` and ``production`` flex-model fields for the ``StorageScheduler`` now act as output sensors: the scheduler writes the resulting power schedule to those sensors, with unit conversion and resampling applied. When only one of the two is defined, the full power profile is written (sign-inverted for the production sensor). When both are defined, the schedule is split into its non-negative (consumption) and non-positive (production) parts. +- Added a ``sign-convention`` query parameter to ``GET /sensors//schedules/`` with three modes: ``consumption-positive`` (default, unchanged behaviour), ``production-positive``, and ``wysiwyg`` (raw database values without sign adjustment). v3.0-30 | 2026-04-15 """""""""""""""""""" diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index f814722710..63781a557c 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -303,4 +303,9 @@ We'd recommend to use positive power values to indicate consumption and negative Read more at :ref:`signs_of_power_beliefs` about our treatment of data, which includes data you send in, or you get from forecasts and schedules (hint: you are free to define the sign for your data, but it might affect how you receive your schedules). -Note that the ``GET /api/v3_0/sensors//schedules/`` endpoint always returns schedules with consumption as positive values and production as negative values. +The ``GET /api/v3_0/sensors//schedules/`` endpoint supports three sign conventions via the ``sign-convention`` query parameter: + +- ``consumption-positive`` (**default**): schedules are returned with consumption as positive values and production as negative values, regardless of how they are stored in the database. +- ``production-positive``: schedules are returned with production as positive values and consumption as negative values. +- ``wysiwyg`` (*what-you-see-is-what-you-get*): the raw database values are returned without any sign adjustment. + The values reflect exactly what is stored, which is governed by the sensor's ``consumption_is_positive`` attribute (if present) or the scheduler's default convention (production positive in the database). diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index c1734cf831..624b77e550 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -172,7 +172,11 @@ Note that, if forecasts are created, they will have the same sign as original da For schedules, the sign of resulting power data (beliefs) is being switched when data is stored (assuming production is positive), and you can prevent that by setting ``sensor.attributes["consumption_is_positive"] = True``. On sensors that have been referenced in a flex-model under the ``consumption`` or ``production`` field, this attribute is automatically set. -The ``GET /api/v3_0/sensors//schedules/`` endpoint always returns schedules with consumption as positive values and production as negative values. +The ``GET /api/v3_0/sensors//schedules/`` endpoint supports three sign conventions via the ``sign-convention`` query parameter: + +- ``consumption-positive`` (default): schedules are always returned with consumption as positive values and production as negative values. +- ``production-positive``: schedules are returned with production as positive values and consumption as negative values. +- ``wysiwyg``: the raw database values are returned without sign adjustment, reflecting exactly what the scheduler stored. .. note:: We will soon document better what the scheduler does in detail, and how the attribute works. From 3e7b8189b1ce4e75cd0c918f50041c37328945aa Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 10:33:13 +0200 Subject: [PATCH 34/44] =?UTF-8?q?api/v3=5F0/tests:=20extend=20sign-convent?= =?UTF-8?q?ion=20tests=20to=20cover=20all=20three=20modes=20Context:=20-?= =?UTF-8?q?=20Both=20sign-convention=20test=20functions=20only=20covered?= =?UTF-8?q?=20the=20default=20=20=20consumption-positive=20convention=20(n?= =?UTF-8?q?o=20sign-convention=20parameter=20was=20passed).=20Change:=20-?= =?UTF-8?q?=20Add=20sign=5Fconvention=20as=20a=20parametrize=20dimension?= =?UTF-8?q?=20(3=20values)=20to=20both=20=20=20test=5Fget=5Fschedule=5Fsig?= =?UTF-8?q?n=5Fconvention=5Fjson=5Fflex=5Fmodel=20and=20=20=20test=5Fget?= =?UTF-8?q?=5Fschedule=5Fsign=5Fconvention=5Fdb=5Fflex=5Fmodel=20(2=20?= =?UTF-8?q?=C3=97=202=20=C3=97=203=20=3D=2012=20cases=20each).=20-=20Extra?= =?UTF-8?q?ct=20=5Fassert=5Fschedule=5Fsign=5Fconvention()=20helper=20with?= =?UTF-8?q?=20a=20documented=20=20=20decision=20table=20for=20expected=20s?= =?UTF-8?q?ign=20vs=20the=20consumption-positive=20reference:=20=20=20=20?= =?UTF-8?q?=20consumption=20+=20consumption-positive=20=E2=86=92=20same=20?= =?UTF-8?q?sign=20as=20main=20=20=20=20=20consumption=20+=20production-pos?= =?UTF-8?q?itive=20=E2=86=92=20opposite=20sign=20=20=20=20=20consumption?= =?UTF-8?q?=20+=20wysiwyg=20=20=20=20=20=20=20=20=20=20=20=20=E2=86=92=20s?= =?UTF-8?q?ame=20sign=20(DB=20is=20consumption=20positive)=20=20=20=20=20p?= =?UTF-8?q?roduction=20=20+=20consumption-positive=20=E2=86=92=20same=20si?= =?UTF-8?q?gn=20=20=20=20=20production=20=20+=20production-positive=20=20?= =?UTF-8?q?=E2=86=92=20opposite=20sign=20=20=20=20=20production=20=20+=20w?= =?UTF-8?q?ysiwyg=20=20=20=20=20=20=20=20=20=20=20=20=20=20=E2=86=92=20opp?= =?UTF-8?q?osite=20sign=20(DB=20is=20production=20positive)=20-=20Add=20fo?= =?UTF-8?q?rce-new-job-creation=20to=20the=20JSON=20flex-model=20trigger?= =?UTF-8?q?=20to=20avoid=20Redis=20=20=20job-cache=20collisions=20across?= =?UTF-8?q?=20parametrized=20runs.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_sensor_schedules_fresh_db.py | 162 +++++++++++------- 1 file changed, 96 insertions(+), 66 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index 50101ca013..601237dc6e 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -16,6 +16,7 @@ handle_scheduling_exception, get_data_source_for_job, ) +from flexmeasures.data.schemas.scheduling import ScheduleSignConvention from flexmeasures.utils.calculations import integrate_time_series @@ -733,6 +734,69 @@ def test_multiple_contracts( ) +def _assert_schedule_sign_convention( + output_type: str, + sign_convention: str, + main_values: list, + output_values: list, +) -> None: + """Assert that *output_values* satisfy the expected sign convention relative to *main_values*. + + *main_values* are always fetched with the default ``consumption-positive`` convention so that + positive values represent charging and negative values represent discharging. + + The expected sign of *output_values* depends on both ``output_type`` and ``sign_convention``: + + - ``consumption`` output sensor stores consumption as positive (``consumption_is_positive=True``): + - ``consumption-positive``: same as DB → same sign as main. + - ``production-positive``: inverts DB → opposite sign from main. + - ``wysiwyg``: raw DB (consumption positive) → same sign as main. + - ``production`` output sensor stores production as positive (``consumption_is_positive=False``): + - ``consumption-positive``: inverts DB → same sign as main. + - ``production-positive``: raw DB (production positive) → opposite sign from main. + - ``wysiwyg``: raw DB (production positive) → opposite sign from main. + """ + same_sign_as_main = ( + sign_convention == ScheduleSignConvention.CONSUMPTION_POSITIVE + or ( + output_type == "consumption" + and sign_convention == ScheduleSignConvention.WYSIWYG + ) + ) + + for main_val, out_val in zip(main_values, output_values): + if main_val > 0: + if same_sign_as_main: + assert out_val >= 0, ( + f"[output_type={output_type}, sign_convention={sign_convention}] " + f"Expected output >= 0 for charging (main={main_val}), got {out_val}" + ) + else: + assert out_val <= 0, ( + f"[output_type={output_type}, sign_convention={sign_convention}] " + f"Expected output <= 0 for charging (main={main_val}), got {out_val}" + ) + elif main_val < 0: + if same_sign_as_main: + assert out_val <= 0, ( + f"[output_type={output_type}, sign_convention={sign_convention}] " + f"Expected output <= 0 for discharging (main={main_val}), got {out_val}" + ) + else: + assert out_val >= 0, ( + f"[output_type={output_type}, sign_convention={sign_convention}] " + f"Expected output >= 0 for discharging (main={main_val}), got {out_val}" + ) + + +@pytest.mark.parametrize( + "sign_convention", + [ + ScheduleSignConvention.CONSUMPTION_POSITIVE, + ScheduleSignConvention.PRODUCTION_POSITIVE, + ScheduleSignConvention.WYSIWYG, + ], +) @pytest.mark.parametrize( "output_type", ["consumption", "production"], @@ -748,13 +812,16 @@ def test_get_schedule_sign_convention_json_flex_model( battery_soc_sensor_fresh_db, keep_scheduling_queue_empty, output_type, + sign_convention, requesting_user, ): - """Test that get_schedule returns correct sign conventions for consumption/production - output sensors when the flex-model is passed via JSON body. + """Test that get_schedule returns correct values for all three sign-convention modes + when the schedule was triggered with a consumption/production sensor in the JSON flex-model. - Consumption output sensor: consumption positive, production negative. - Production output sensor: consumption positive, production negative. + The main power sensor is used as a reference (always fetched with the default + ``consumption-positive`` convention). The expected sign of the output sensor values + relative to that reference depends on ``output_type`` × ``sign_convention``; see + :func:`_assert_schedule_sign_convention` for the full decision table. """ battery_asset = add_battery_assets_fresh_db["Test battery"] @@ -768,8 +835,12 @@ def test_get_schedule_sign_convention_json_flex_model( fresh_db.session.add(output_sensor) fresh_db.session.flush() - # Build the trigger message with the output sensor in the flex-model + # Build the trigger message with the output sensor in the flex-model. + # force-new-job-creation bypasses the Redis job cache, which would otherwise return a cached + # job_id from the previous parametrized run (fresh_db resets sensor IDs to the same values + # each function, producing the same trigger hash). message = message_for_trigger_schedule(resolution="PT1H") + message["force-new-job-creation"] = True price_sensor_id = add_market_prices_fresh_db["epex_da"].id message["flex-context"] = { "consumption-price": {"sensor": price_sensor_id}, @@ -799,20 +870,19 @@ def test_get_schedule_sign_convention_json_flex_model( job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) assert job.is_finished - # Retrieve the schedule from the output sensor + # Retrieve the schedule from the output sensor using the requested sign convention with app.test_client() as client: get_response = client.get( url_for("SensorAPI:get_schedule", id=output_sensor.id, uuid=job_id), - query_string={"duration": "PT24H"}, + query_string={"duration": "PT24H", "sign-convention": sign_convention}, ) assert get_response.status_code == 200 values = get_response.json["values"] - # The battery schedule should have non-zero values (charging and/or discharging) assert len(values) > 0 assert any(v != 0 for v in values), "Schedule should have non-zero values" - # Get the main power sensor schedule for comparison (consumption positive convention) + # Get the main power sensor schedule using the default convention as reference with app.test_client() as client: main_response = client.get( url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), @@ -821,37 +891,17 @@ def test_get_schedule_sign_convention_json_flex_model( assert main_response.status_code == 200 main_values = main_response.json["values"] - if output_type == "consumption": - # Consumption sensor returns the full schedule in consumption-positive convention - # (same sign as main power sensor schedule) - # Both should have the same absolute pattern, consumption positive - for main_val, out_val in zip(main_values, values): - if main_val > 0: - # Charging (consumption) should be positive on consumption sensor - assert ( - out_val >= 0 - ), f"Consumption sensor should show positive for charging: main={main_val}, output={out_val}" - elif main_val < 0: - # Discharging (production) should be negative on consumption sensor - assert ( - out_val <= 0 - ), f"Consumption sensor should show negative for discharging: main={main_val}, output={out_val}" - elif output_type == "production": - # Production sensor returns the full schedule in consumption-positive convention - # (same sign as main power sensor schedule) - for main_val, out_val in zip(main_values, values): - if main_val > 0: - # Charging (consumption) should be positive on production sensor - assert ( - out_val >= 0 - ), f"Production sensor should show positive for charging: main={main_val}, output={out_val}" - elif main_val < 0: - # Discharging (production) should be negative on production sensor - assert ( - out_val <= 0 - ), f"Production sensor should show negative for discharging: main={main_val}, output={out_val}" + _assert_schedule_sign_convention(output_type, sign_convention, main_values, values) +@pytest.mark.parametrize( + "sign_convention", + [ + ScheduleSignConvention.CONSUMPTION_POSITIVE, + ScheduleSignConvention.PRODUCTION_POSITIVE, + ScheduleSignConvention.WYSIWYG, + ], +) @pytest.mark.parametrize( "output_type", ["consumption", "production"], @@ -867,13 +917,13 @@ def test_get_schedule_sign_convention_db_flex_model( battery_soc_sensor_fresh_db, keep_scheduling_queue_empty, output_type, + sign_convention, requesting_user, ): - """Test that get_schedule returns correct sign conventions for consumption/production - output sensors when the flex-model is stored in the DB (asset attributes). + """Test that get_schedule returns correct values for all three sign-convention modes + when the consumption/production sensor is stored in the DB flex-model (asset attributes). - Consumption output sensor: consumption positive, production negative. - Production output sensor: production positive, consumption negative. + See :func:`_assert_schedule_sign_convention` for the full decision table. """ battery_asset = add_battery_assets_fresh_db["Test battery"] @@ -927,22 +977,21 @@ def test_get_schedule_sign_convention_db_flex_model( job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) assert job.is_finished - # Retrieve the schedule from the output sensor + # Retrieve the schedule from the output sensor using the requested sign convention with app.test_client() as client: get_response = client.get( url_for("SensorAPI:get_schedule", id=output_sensor.id, uuid=job_id), - query_string={"duration": "PT24H"}, + query_string={"duration": "PT24H", "sign-convention": sign_convention}, ) assert ( get_response.status_code == 200 ), f"GET schedule failed: {get_response.json}" values = get_response.json["values"] - # The battery schedule should have non-zero values assert len(values) > 0 assert any(v != 0 for v in values), "Schedule should have non-zero values" - # Get the main power sensor schedule for comparison + # Get the main power sensor schedule using the default convention as reference with app.test_client() as client: main_response = client.get( url_for("SensorAPI:get_schedule", id=sensor.id, uuid=job_id), @@ -951,26 +1000,7 @@ def test_get_schedule_sign_convention_db_flex_model( assert main_response.status_code == 200 main_values = main_response.json["values"] - if output_type == "consumption": - for main_val, out_val in zip(main_values, values): - if main_val > 0: - assert ( - out_val >= 0 - ), f"Consumption sensor should show positive for charging: main={main_val}, output={out_val}" - elif main_val < 0: - assert ( - out_val <= 0 - ), f"Consumption sensor should show negative for discharging: main={main_val}, output={out_val}" - elif output_type == "production": - for main_val, out_val in zip(main_values, values): - if main_val > 0: - assert ( - out_val >= 0 - ), f"Production sensor should show positive for charging: main={main_val}, output={out_val}" - elif main_val < 0: - assert ( - out_val <= 0 - ), f"Production sensor should show negative for discharging: main={main_val}, output={out_val}" + _assert_schedule_sign_convention(output_type, sign_convention, main_values, values) # Clean up: remove the flex_model entry so it doesn't affect other tests del battery_asset.flex_model[output_type] From 5eb177275f1caa9146680fa5f34c7e16ee9e96ce Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 10:36:09 +0200 Subject: [PATCH 35/44] docs: touch up sign explanations Signed-off-by: F.N. Claessen --- documentation/api/notation.rst | 2 +- documentation/concepts/data-model.rst | 8 +++----- flexmeasures/api/v3_0/sensors.py | 2 +- flexmeasures/ui/static/openapi-specs.json | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index 63781a557c..b9ffb01440 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -308,4 +308,4 @@ The ``GET /api/v3_0/sensors//schedules/`` endpoint supports three sign - ``consumption-positive`` (**default**): schedules are returned with consumption as positive values and production as negative values, regardless of how they are stored in the database. - ``production-positive``: schedules are returned with production as positive values and consumption as negative values. - ``wysiwyg`` (*what-you-see-is-what-you-get*): the raw database values are returned without any sign adjustment. - The values reflect exactly what is stored, which is governed by the sensor's ``consumption_is_positive`` attribute (if present) or the scheduler's default convention (production positive in the database). + The values reflect exactly what is stored, which is itself governed by the sensor's ``consumption_is_positive`` attribute (if present) or the scheduler's default convention (production positive in the database). diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index 624b77e550..f619c799f4 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -168,18 +168,16 @@ For example, users can create PV power data with positive values indicating prod We allow this because we want the UI to match what is in the database, and users often desire both of these datasets to be shown as positive values. We assume that this is what users send in. -Note that, if forecasts are created, they will have the same sign as original data. +Note that, if forecasts are created, they will have the same sign as the original data. For schedules, the sign of resulting power data (beliefs) is being switched when data is stored (assuming production is positive), and you can prevent that by setting ``sensor.attributes["consumption_is_positive"] = True``. On sensors that have been referenced in a flex-model under the ``consumption`` or ``production`` field, this attribute is automatically set. + The ``GET /api/v3_0/sensors//schedules/`` endpoint supports three sign conventions via the ``sign-convention`` query parameter: - ``consumption-positive`` (default): schedules are always returned with consumption as positive values and production as negative values. - ``production-positive``: schedules are returned with production as positive values and consumption as negative values. -- ``wysiwyg``: the raw database values are returned without sign adjustment, reflecting exactly what the scheduler stored. - - -.. note:: We will soon document better what the scheduler does in detail, and how the attribute works. +- ``wysiwyg`` (*what-you-see-is-what-you-get*): the raw database values are returned without sign adjustment, reflecting exactly what the scheduler stored. Accounts & Users diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index af33f76cc0..a62352cc01 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1123,7 +1123,7 @@ def get_schedule( # noqa: C901 Sign convention applied to power values in the response. - ``consumption-positive`` (default): consumption is positive, production is negative. - ``production-positive``: production is positive, consumption is negative. - - ``wysiwyg``: raw database values returned without sign adjustment. + - ``wysiwyg`` (*what-you-see-is-what-you-get*): raw database values returned without sign adjustment. example: consumption-positive schema: type: string diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 40188b6480..50e24d9c9f 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -741,7 +741,7 @@ "in": "query", "name": "sign-convention", "required": false, - "description": "Sign convention applied to power values in the response.\n- consumption-positive (default): consumption is positive, production is negative.\n- production-positive: production is positive, consumption is negative.\n- wysiwyg: raw database values returned without sign adjustment.\n", + "description": "Sign convention applied to power values in the response.\n- consumption-positive (default): consumption is positive, production is negative.\n- production-positive: production is positive, consumption is negative.\n- wysiwyg (what-you-see-is-what-you-get): raw database values returned without sign adjustment.\n", "example": "consumption-positive", "schema": { "type": "string", From 99f5d90a9f78ec36f35d789b2a99f0b791477140 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 10:49:11 +0200 Subject: [PATCH 36/44] docs: clarify how the scheduler records the data, mention the sign of database values, rather than 'raw database values', and stop using the term 'reflect', which itself suggest a sign flip Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 2 +- documentation/api/notation.rst | 4 ++-- documentation/concepts/data-model.rst | 17 +++++++++++++---- flexmeasures/api/v3_0/sensors.py | 18 +++++++++--------- .../data/schemas/scheduling/__init__.py | 4 ++-- flexmeasures/ui/static/openapi-specs.json | 4 ++-- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index c55d79f149..a1e9833455 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -12,7 +12,7 @@ v3.0-31 | 2026-05-20 - Switched from ``force_new_job_creation`` to ``force-new-job-creation`` (maintaining backwards compatibility) and added the field to `/assets/(id)/schedules/trigger` (POST) endpoint, too. - Support both snake_case and kebab-case fields in `/sensors//data` (GET), while only documenting the kebab-case ones. - The ``consumption`` and ``production`` flex-model fields for the ``StorageScheduler`` now act as output sensors: the scheduler writes the resulting power schedule to those sensors, with unit conversion and resampling applied. When only one of the two is defined, the full power profile is written (sign-inverted for the production sensor). When both are defined, the schedule is split into its non-negative (consumption) and non-positive (production) parts. -- Added a ``sign-convention`` query parameter to ``GET /sensors//schedules/`` with three modes: ``consumption-positive`` (default, unchanged behaviour), ``production-positive``, and ``wysiwyg`` (raw database values without sign adjustment). +- Added a ``sign-convention`` query parameter to ``GET /sensors//schedules/`` with three modes: ``consumption-positive`` (default, unchanged behaviour), ``production-positive``, and ``wysiwyg`` (sign of database values and as seen in UI charts). v3.0-30 | 2026-04-15 """""""""""""""""""" diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index b9ffb01440..a00aaeedd4 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -307,5 +307,5 @@ The ``GET /api/v3_0/sensors//schedules/`` endpoint supports three sign - ``consumption-positive`` (**default**): schedules are returned with consumption as positive values and production as negative values, regardless of how they are stored in the database. - ``production-positive``: schedules are returned with production as positive values and consumption as negative values. -- ``wysiwyg`` (*what-you-see-is-what-you-get*): the raw database values are returned without any sign adjustment. - The values reflect exactly what is stored, which is itself governed by the sensor's ``consumption_is_positive`` attribute (if present) or the scheduler's default convention (production positive in the database). +- ``wysiwyg`` (*what-you-see-is-what-you-get*): schedules are returned with the same sign as database values and as seen in the UI charts. + The values indicate exactly what is stored, which is itself governed by the sensor's ``consumption_is_positive`` attribute (if present) or the scheduler's default convention (production positive in the database). diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index f619c799f4..94a5ae2f48 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -79,6 +79,7 @@ A data source can be a FlexMeasures user, but also simply a named source from ou In FlexMeasures, data sources have a type. It is just a string which you can freely choose (we do not model them explicitly im the data model like Asset types). We do support some types out of the box: "scheduler", "forecaster" "reporter", "demo script" and "user". +.. _beliefs: Beliefs --------- @@ -156,7 +157,8 @@ More information, including code examples, is available in :ref:`annotations`. About signs of power & energy values ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In short: You can use any sign you want for power data. -But the scheduler in FlexMeasures needs to know how to apply the signs. Positive (+) means consumption, negative (-) means production. +What is recorded in the database is exactly as seen in UI charts. +But the scheduler in FlexMeasures needs to know how to apply the signs. Let us explain. When beliefs are about power or energy, the sign of the value is important. It indicates whether the asset is consuming or producing. @@ -170,14 +172,21 @@ We assume that this is what users send in. Note that, if forecasts are created, they will have the same sign as the original data. -For schedules, the sign of resulting power data (beliefs) is being switched when data is stored (assuming production is positive), and you can prevent that by setting ``sensor.attributes["consumption_is_positive"] = True``. -On sensors that have been referenced in a flex-model under the ``consumption`` or ``production`` field, this attribute is automatically set. +For schedules, the sign of the power schedule (as :ref:`beliefs `) recorded in the database, and as seen in UI charts, is determined as follows: + +- If the flex-model contains the ``sensor`` field, and that sensor has power units (e.g. kW), the ``"consumption_is_positive"`` attribute of the sensor is used to decide the sign of the recorded data. + If the attribute is not defined, **by default, scheduled power is recorded with production as positive values** (and consumption as negative values). + To record scheduled power data with consumption as positive values, set ``sensor.attributes["consumption_is_positive"] = True``. +- If the flex-model contains the ``consumption`` field, scheduled power is recorded with consumption as positive values. + The ``"consumption_is_positive"`` attribute of the referenced sensor is set automatically to ``True``. +- If the flex-model contains the ``production`` field, scheduled power is recorded with production as positive values. + The ``"consumption_is_positive"`` attribute of the referenced sensor is set automatically to ``False``. The ``GET /api/v3_0/sensors//schedules/`` endpoint supports three sign conventions via the ``sign-convention`` query parameter: - ``consumption-positive`` (default): schedules are always returned with consumption as positive values and production as negative values. - ``production-positive``: schedules are returned with production as positive values and consumption as negative values. -- ``wysiwyg`` (*what-you-see-is-what-you-get*): the raw database values are returned without sign adjustment, reflecting exactly what the scheduler stored. +- ``wysiwyg`` (*what-you-see-is-what-you-get*): schedules are returned with the same sign as database values and as seen in the UI charts, indicating exactly what the scheduler stored. Accounts & Users diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index a62352cc01..e50bf2387d 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1065,16 +1065,16 @@ def get_schedule( # noqa: C901 **Sign convention** By default (``sign-convention: consumption-positive``), the endpoint always returns schedules where - consumption is positive and production is negative, regardless of how the values are stored in the - database. This is the most common convention and matches the perspective of a consumer. + consumption is positive and production is negative, regardless of how the values are stored in the database. + This is the most common convention and matches the perspective of a consumer. Set ``sign-convention: production-positive`` to flip the sign so that production is returned as positive and consumption as negative. This matches the perspective of a producer. - Set ``sign-convention: wysiwyg`` (*what-you-see-is-what-you-get*) to return the raw database values - without any sign adjustment. The values will reflect exactly what is stored, which is determined - by the sensor's ``consumption_is_positive`` attribute (if set) or by the scheduler's default - storage convention (production positive in the database). + Set ``sign-convention: wysiwyg`` (*what-you-see-is-what-you-get*) to return the values with the same sign + as database values and what is seen in UI charts. The values will indicate exactly what is stored, + which is itself determined by the sensor's ``consumption_is_positive`` attribute (if set) + or by the scheduler's default storage convention (production positive in the database). security: - ApiKeyAuth: [] parameters: @@ -1123,7 +1123,7 @@ def get_schedule( # noqa: C901 Sign convention applied to power values in the response. - ``consumption-positive`` (default): consumption is positive, production is negative. - ``production-positive``: production is positive, consumption is negative. - - ``wysiwyg`` (*what-you-see-is-what-you-get*): raw database values returned without sign adjustment. + - ``wysiwyg`` (*what-you-see-is-what-you-get*): sign of database values and as seen in UI charts. example: consumption-positive schema: type: string @@ -1258,8 +1258,8 @@ def get_schedule( # noqa: C901 sign = 1 if sign_convention == ScheduleSignConvention.WYSIWYG: - # Return raw database values without sign adjustment. - # No sign inversion: what's in the DB is what the caller receives. + # Return values without adjusting the sign of database values. + # No sign inversion: what's in the DB and what is seen in UI charts is what the caller receives. pass elif sensor.measures_power: # Determine whether the database stores consumption as positive or negative. diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index e99b2731d2..38ab39a340 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1057,8 +1057,8 @@ class GetScheduleSchema(Schema): f"and production as negative values. " f"``{ScheduleSignConvention.PRODUCTION_POSITIVE}``: production is always returned as positive values " f"and consumption as negative values. " - f"``{ScheduleSignConvention.WYSIWYG}``: the raw database values are returned without " - "any sign adjustment, regardless of the sensor's ``consumption_is_positive`` attribute." + f"``{ScheduleSignConvention.WYSIWYG}``: returns values with the same sign as database values and as seen in the UI charts, " + "without adjusting their sign for the sensor's ``consumption_is_positive`` attribute." ), example=ScheduleSignConvention.CONSUMPTION_POSITIVE, ), diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 50e24d9c9f..c0ae18e28f 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -690,7 +690,7 @@ "/api/v3_0/sensors/{id}/schedules/{uuid}": { "get": { "summary": "Get schedule for one device", - "description": "Get a schedule from FlexMeasures.\n\nOptional fields:\n\n- \"duration\" (6 hours by default; can be increased to plan further into the future)\n- \"unit\" (by default, the unit of the schedule is the sensor's unit; a compatible unit can be requested)\n- \"sign-convention\" (controls how power values are signed in the response; see below)\n\nSign convention\n\nBy default (sign-convention: consumption-positive), the endpoint always returns schedules where\nconsumption is positive and production is negative, regardless of how the values are stored in the\ndatabase. This is the most common convention and matches the perspective of a consumer.\n\nSet sign-convention: production-positive to flip the sign so that production is returned as\npositive and consumption as negative. This matches the perspective of a producer.\n\nSet sign-convention: wysiwyg (what-you-see-is-what-you-get) to return the raw database values\nwithout any sign adjustment. The values will reflect exactly what is stored, which is determined\nby the sensor's consumption_is_positive attribute (if set) or by the scheduler's default\nstorage convention (production positive in the database).\n", + "description": "Get a schedule from FlexMeasures.\n\nOptional fields:\n\n- \"duration\" (6 hours by default; can be increased to plan further into the future)\n- \"unit\" (by default, the unit of the schedule is the sensor's unit; a compatible unit can be requested)\n- \"sign-convention\" (controls how power values are signed in the response; see below)\n\nSign convention\n\nBy default (sign-convention: consumption-positive), the endpoint always returns schedules where\nconsumption is positive and production is negative, regardless of how the values are stored in the database.\nThis is the most common convention and matches the perspective of a consumer.\n\nSet sign-convention: production-positive to flip the sign so that production is returned as\npositive and consumption as negative. This matches the perspective of a producer.\n\nSet sign-convention: wysiwyg (what-you-see-is-what-you-get) to return the values with the same sign\nas database values and what is seen in UI charts. The values will indicate exactly what is stored,\nwhich is itself determined by the sensor's consumption_is_positive attribute (if set)\nor by the scheduler's default storage convention (production positive in the database).\n", "security": [ { "ApiKeyAuth": [] @@ -741,7 +741,7 @@ "in": "query", "name": "sign-convention", "required": false, - "description": "Sign convention applied to power values in the response.\n- consumption-positive (default): consumption is positive, production is negative.\n- production-positive: production is positive, consumption is negative.\n- wysiwyg (what-you-see-is-what-you-get): raw database values returned without sign adjustment.\n", + "description": "Sign convention applied to power values in the response.\n- consumption-positive (default): consumption is positive, production is negative.\n- production-positive: production is positive, consumption is negative.\n- wysiwyg (what-you-see-is-what-you-get): sign of database values and as seen in UI charts.\n", "example": "consumption-positive", "schema": { "type": "string", From e29111c589719b83197a59fd1ccdbe9cff761dd7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 12:36:42 +0200 Subject: [PATCH 37/44] docs: clarify confusing comments Signed-off-by: F.N. Claessen --- documentation/api/notation.rst | 2 +- flexmeasures/data/models/planning/storage.py | 11 ++++++----- flexmeasures/data/schemas/scheduling/metadata.py | 8 ++++---- flexmeasures/data/services/scheduling.py | 5 +++-- flexmeasures/ui/static/openapi-specs.json | 4 ++-- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index a00aaeedd4..969ff22f92 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -308,4 +308,4 @@ The ``GET /api/v3_0/sensors//schedules/`` endpoint supports three sign - ``consumption-positive`` (**default**): schedules are returned with consumption as positive values and production as negative values, regardless of how they are stored in the database. - ``production-positive``: schedules are returned with production as positive values and consumption as negative values. - ``wysiwyg`` (*what-you-see-is-what-you-get*): schedules are returned with the same sign as database values and as seen in the UI charts. - The values indicate exactly what is stored, which is itself governed by the sensor's ``consumption_is_positive`` attribute (if present) or the scheduler's default convention (production positive in the database). + The values indicate exactly what is stored, which is itself governed by the sensor's ``consumption_is_positive`` attribute (if present) or the scheduler's default convention (to store production as positive values in the database). diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index f724e02fef..5d776e5015 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1600,16 +1600,17 @@ def _build_consumption_production_schedules( The schedule stored on each sensor depends on which sensors are defined: - **Only** ``consumption`` **sensor defined**: the full power schedule is written to that - sensor using the standard FlexMeasures sign convention (consumption positive, production - negative). + sensor using the consumption-positive sign convention (consumption positive, production + negative), which already corresponds to the sign convention of the scheduler. - **Only** ``production`` **sensor defined**: the full power schedule is written to that - sensor with the sign inverted (production positive, consumption negative). + sensor with production-positive convention (production positive, consumption negative). + The sign is inverted with respect to the sign convention of the scheduler. - **Both** ``consumption`` **and** ``production`` **sensors defined**: only the non-negative part of the schedule is written to the consumption sensor, and only the non-positive part (sign-flipped to positive values) is written to the production sensor. - Because the sign convention is encoded in the sensor key name (``consumption`` vs. - ``production``), these sensors do not need a ``consumption_is_positive`` attribute. + The sign convention is determined by the key name, + and is stored on the sensor itself using the ``consumption_is_positive`` attribute. Unit conversion from MW to each sensor's unit is applied. diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 2df2201388..2aaa23b9a1 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -188,14 +188,14 @@ def to_dict(self): CONSUMPTION = MetaData( description="""Sensor used to record the scheduled power as seen from a consumption perspective. -The sign convention is determined by the key name, so the sensor itself does not need a ``consumption_is_positive`` attribute. +The sign convention is determined by the key name, and is stored on the sensor itself using the ``consumption_is_positive`` attribute. Depending on which output sensors are defined: - **Only** ``consumption`` **defined**: the full power schedule is stored on this sensor using the - standard FlexMeasures sign convention (consumption positive, production negative). + consumption-positive sign convention (consumption positive, production negative). - **Only** ``production`` **defined**: the full power schedule is stored on the production sensor - with the sign inverted (production positive, consumption negative). + with the production-positive convention (production positive, consumption negative). - **Both defined**: only the non-negative part of the schedule is stored on this sensor (zero for time steps with net production), and only the non-positive part (sign-flipped) is stored on the production sensor. @@ -205,7 +205,7 @@ def to_dict(self): PRODUCTION = MetaData( description="""Sensor used to record the scheduled power as seen from a production perspective. -The sign convention is determined by the key name, so the sensor itself does not need a ``consumption_is_positive`` attribute. +The sign convention is determined by the key name, and is stored on the sensor itself using the ``consumption_is_positive`` attribute. See ``consumption`` for the full description of the split logic when both sensors are defined. """, diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 561d67a1a8..4a7fbe2c7e 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -601,8 +601,9 @@ def _resolve_schedule_output_sign( """Determine the sign multiplier for a schedule output result. Returns 1 (no sign change) or -1 (invert sign) depending on whether the result - is a power schedule that needs sign conversion to match FlexMeasures convention - (consumption positive, production negative). + is a power schedule that needs sign conversion to match FlexMeasures' default + of recording production as positive values in the database. + The power schedule itself denotes consumption as positive values, hence the flip. For consumption/production output schedules (identified by result name and sensor), the sign is already correct per the scheduler, so no conversion is applied. diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index c0ae18e28f..277e3c5ed4 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -6015,14 +6015,14 @@ "type": "object", "properties": { "consumption": { - "description": "Sensor used to record the scheduled power as seen from a consumption perspective.\n\nThe sign convention is determined by the key name, so the sensor itself does not need a consumption_is_positive attribute.\n\nDepending on which output sensors are defined:\n\n- Only consumption defined: the full power schedule is stored on this sensor using the\n standard FlexMeasures sign convention (consumption positive, production negative).\n- Only production defined: the full power schedule is stored on the production sensor\n with the sign inverted (production positive, consumption negative).\n- Both defined: only the non-negative part of the schedule is stored on this sensor (zero for\n time steps with net production), and only the non-positive part (sign-flipped) is stored on the\n production sensor.\n", + "description": "Sensor used to record the scheduled power as seen from a consumption perspective.\n\nThe sign convention is determined by the key name, and is stored on the sensor itself using the consumption_is_positive attribute.\n\nDepending on which output sensors are defined:\n\n- Only consumption defined: the full power schedule is stored on this sensor using the\n consumption-positive sign convention (consumption positive, production negative).\n- Only production defined: the full power schedule is stored on the production sensor\n with the production-positive convention (production positive, consumption negative).\n- Both defined: only the non-negative part of the schedule is stored on this sensor (zero for\n time steps with net production), and only the non-positive part (sign-flipped) is stored on the\n production sensor.\n", "example": { "sensor": 14 }, "$ref": "#/components/schemas/SensorReference" }, "production": { - "description": "Sensor used to record the scheduled power as seen from a production perspective.\n\nThe sign convention is determined by the key name, so the sensor itself does not need a consumption_is_positive attribute.\n\nSee consumption for the full description of the split logic when both sensors are defined.\n", + "description": "Sensor used to record the scheduled power as seen from a production perspective.\n\nThe sign convention is determined by the key name, and is stored on the sensor itself using the consumption_is_positive attribute.\n\nSee consumption for the full description of the split logic when both sensors are defined.\n", "example": { "sensor": 15 }, From 7e6ab5102107772e8c4b6b07153034cf82fbe7f2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 13:19:32 +0200 Subject: [PATCH 38/44] data/models/planning/storage: pass consumption-positive values to make_schedule for all output sensor cases Context: - Previously _build_consumption_production_schedules flipped the sign for the production-only case (-power_series) and for the both-sensors case ((-power_series).clip(lower=0)), forcing _resolve_schedule_output_sign to bypass its inversion logic for dedicated output sensors. Change: - Production-only case: pass power_series unchanged (consumption positive); make_schedule inverts via consumption_is_positive=False on the sensor. - Both-sensors case: clip production to power_series.clip(upper=0) (the non-positive part, still in consumption-positive convention); make_schedule inverts via the same attribute. - Documentation in the docstring updated to reflect the new flow. --- flexmeasures/data/models/planning/storage.py | 30 ++++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 5d776e5015..db8395c1b5 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1600,17 +1600,21 @@ def _build_consumption_production_schedules( The schedule stored on each sensor depends on which sensors are defined: - **Only** ``consumption`` **sensor defined**: the full power schedule is written to that - sensor using the consumption-positive sign convention (consumption positive, production - negative), which already corresponds to the sign convention of the scheduler. + sensor using the scheduler's native sign convention (consumption positive, production + negative). ``make_schedule`` applies no further sign change because the sensor already + has ``consumption_is_positive=True``. - **Only** ``production`` **sensor defined**: the full power schedule is written to that - sensor with production-positive convention (production positive, consumption negative). - The sign is inverted with respect to the sign convention of the scheduler. + sensor in the scheduler's native sign convention (consumption positive, production + negative). ``make_schedule`` inverts the sign based on the sensor's + ``consumption_is_positive=False`` attribute so that production is stored as positive values. - **Both** ``consumption`` **and** ``production`` **sensors defined**: only the non-negative - part of the schedule is written to the consumption sensor, and only the non-positive part - (sign-flipped to positive values) is written to the production sensor. + part of the schedule (charging / consuming) is written to the consumption sensor, and only + the non-positive part (discharging / producing, still as negative values) is written to + the production sensor. ``make_schedule`` inverts the sign for the production sensor. - The sign convention is determined by the key name, - and is stored on the sensor itself using the ``consumption_is_positive`` attribute. + The ``consumption_is_positive`` attribute is set on each output sensor when the scheduling + job is created (see ``create_scheduling_job``), not here. This method only clips the + series; sign handling is left entirely to ``make_schedule``. Unit conversion from MW to each sensor's unit is applied. @@ -1644,15 +1648,17 @@ def _build_consumption_production_schedules( event_resolution=consumption_sensor.event_resolution, ) elif production_sensor is not None and consumption_sensor is None: - # Full power profile on the production sensor (production positive, consumption negative). + # Full power profile on the production sensor in native scheduler convention. + # make_schedule inverts the sign via consumption_is_positive=False on the sensor. schedules[production_sensor] = convert_units( - -power_series, + power_series, "MW", production_sensor.unit, event_resolution=production_sensor.event_resolution, ) else: - # Both sensors defined: split into non-negative (consumption) and non-positive (production) parts. + # Both sensors defined: clip to non-negative (consumption) and non-positive (production) parts. + # make_schedule inverts the sign for the production sensor via consumption_is_positive=False. schedules[consumption_sensor] = convert_units( power_series.clip(lower=0), "MW", @@ -1660,7 +1666,7 @@ def _build_consumption_production_schedules( event_resolution=consumption_sensor.event_resolution, ) schedules[production_sensor] = convert_units( - (-power_series).clip(lower=0), + power_series.clip(upper=0), "MW", production_sensor.unit, event_resolution=production_sensor.event_resolution, From 6905eac987bdab49e2b5892fc10067ef691ab57f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 13:19:45 +0200 Subject: [PATCH 39/44] data/services/scheduling: set consumption_is_positive at trigger time and simplify sign resolution Context: - The _set_output_sensor_consumption_is_positive safety-net ran only inside make_schedule (after scheduling), too late to surface attribute conflicts. - _resolve_schedule_output_sign special-cased dedicated output sensors with an early return of 1, which was only correct because _build_consumption_ production_schedules was flipping signs itself. Change: - Add _set_flex_model_output_sensors_consumption_is_positive() that iterates the deserialized flex model and assigns consumption_is_positive to each output sensor (True for consumption, False for production), raising ValueError immediately on conflict. - Call it in create_scheduling_job() right after deserialize_config(), so attribute conflicts surface as 422 responses before any job is enqueued. - Simplify _resolve_schedule_output_sign(): remove the _is_consumption_ production_output() early-return and let it fall through to the consumption_is_positive attribute check that already handles all cases. - Retain the _set_output_sensor_consumption_is_positive() call inside make_schedule as a safety net for direct invocations (moved before save_to_db for fail-fast behaviour). --- flexmeasures/data/services/scheduling.py | 94 +++++++++++++++++++----- 1 file changed, 77 insertions(+), 17 deletions(-) diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 4a7fbe2c7e..772f51e388 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -254,6 +254,10 @@ def create_scheduling_job( scheduler_kwargs["flex_model"] = scheduler.flex_model scheduler.deserialize_config() + # Set consumption_is_positive on output sensors now (at trigger time) so that any + # attribute conflict raises an error immediately, before the job is enqueued. + _set_flex_model_output_sensors_consumption_is_positive(scheduler.flex_model) + asset_or_sensor = get_asset_or_sensor_ref(asset_or_sensor) job = Job.create( make_schedule, @@ -550,6 +554,58 @@ def _is_consumption_production_output( return not is_main +def _set_flex_model_output_sensors_consumption_is_positive( + flex_model: dict | list, +) -> None: + """Set the ``consumption_is_positive`` attribute on consumption and production output sensors. + + Iterates over every device in *flex_model* and assigns:: + + consumption sensor → consumption_is_positive = True + production sensor → consumption_is_positive = False + + A ``ValueError`` is raised immediately when the attribute is already present on a sensor + but has the wrong value for the flex-model field that references it. Calling this function + at job-creation time lets the API surface conflicts before any work is queued. + + :param flex_model: Deserialized flex model — either a single-device ``dict`` or a + ``list`` of per-device dicts. Consumption/production fields are + expected to be dicts with a ``"sensor"`` key. + :raises ValueError: When ``consumption_is_positive`` is already set to the wrong value + for the given flex-model field. + """ + models = flex_model if isinstance(flex_model, list) else [flex_model] + for flex_model_d in models: + consumption_field = flex_model_d.get("consumption") + production_field = flex_model_d.get("production") + consumption_sensor = ( + consumption_field.get("sensor") + if isinstance(consumption_field, dict) + else None + ) + production_sensor = ( + production_field.get("sensor") + if isinstance(production_field, dict) + else None + ) + for sensor, intended in [ + (consumption_sensor, True), + (production_sensor, False), + ]: + if sensor is None: + continue + field_name = "consumption_schedule" if intended else "production_schedule" + existing = sensor.attributes.get("consumption_is_positive") + if existing is not None and existing != intended: + raise ValueError( + f"Sensor {sensor} already has `consumption_is_positive={existing}`, " + f"which conflicts with the '{field_name}' output schedule " + f"(expected `consumption_is_positive={intended}`). " + f"Remove or correct the attribute before running the scheduler." + ) + sensor.attributes["consumption_is_positive"] = intended + + def _set_output_sensor_consumption_is_positive( result: dict, asset_or_sensor: Asset | Sensor ) -> None: @@ -601,13 +657,20 @@ def _resolve_schedule_output_sign( """Determine the sign multiplier for a schedule output result. Returns 1 (no sign change) or -1 (invert sign) depending on whether the result - is a power schedule that needs sign conversion to match FlexMeasures' default - of recording production as positive values in the database. - The power schedule itself denotes consumption as positive values, hence the flip. + is a power schedule that needs sign conversion so that production is stored as positive + values in the database. - For consumption/production output schedules (identified by result name and sensor), - the sign is already correct per the scheduler, so no conversion is applied. - For other power schedules (main power sensors), the standard conversion is applied. + The scheduler always produces consumption-positive values. For sensors that carry + ``consumption_is_positive=True`` (including consumption output sensors) no conversion + is needed. For sensors with ``consumption_is_positive=False`` (production output sensors + and the default convention for main power sensors) the sign is inverted. + + .. note:: + For consumption/production output sensors the ``consumption_is_positive`` attribute + must be set before this function is called. It is set eagerly at job-creation time + by :func:`_set_flex_model_output_sensors_consumption_is_positive`, and again (as a + safety-net for direct :func:`make_schedule` calls) by + :func:`_set_output_sensor_consumption_is_positive` earlier in the same loop iteration. :param result: Schedule output result dict with keys 'name', 'sensor', 'data'. :param asset_or_sensor: The Asset or Sensor being scheduled (main power sensor). @@ -615,14 +678,8 @@ def _resolve_schedule_output_sign( """ result_sensor = result["sensor"] - # Consumption/production output schedules have their sign convention already - # encoded in the field name ("consumption" = consumption positive; - # "production" = production positive). Only main power schedules need conversion. - if _is_consumption_production_output(result, asset_or_sensor): - return 1 - - # Apply standard sign inversion only for main power schedules that measure power - # and don't have the consumption_is_positive attribute. + # Apply sign inversion for power sensors that record production as positive values + # (i.e. those that do not carry consumption_is_positive=True). if result_sensor.measures_power and not result_sensor.get_attribute( "consumption_is_positive", False ): @@ -746,6 +803,12 @@ def make_schedule( # noqa: C901 if "sensor" not in result: continue + # Ensure consumption_is_positive is set before resolving the sign. + # At job-creation time this is already done eagerly; calling it here again + # acts as a safety net for direct make_schedule invocations and raises a + # ValueError on attribute conflicts before any data are written. + _set_output_sensor_consumption_is_positive(result, asset_or_sensor) + sign = _resolve_schedule_output_sign(result, asset_or_sensor) ts_value_schedule = [ @@ -769,9 +832,6 @@ def make_schedule( # noqa: C901 bdf = bdf.resample_events(bdf.sensor.event_resolution) if not dry_run: - # Validate and set the consumption_is_positive attribute before writing data so - # that a conflict raises an error early, before any beliefs are persisted. - _set_output_sensor_consumption_is_positive(result, asset_or_sensor) save_to_db(bdf) else: print( From 5600f6f5870680462c17a51341d57feb6853cc50 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 13:19:56 +0200 Subject: [PATCH 40/44] api/v3_0/tests: update conflict test to expect 422 at trigger time Context: - Now that consumption_is_positive conflicts are detected at job-creation time, the trigger endpoint returns 422 immediately instead of creating a job that later fails. Change: - Rename test to test_conflicting_consumption_is_positive_attribute_prevents_job_creation. - Remove RQ job execution and job.is_failed check. - Assert trigger_response.status_code == 422 and that the error message contains 'consumption_is_positive'. - Fix assertion to use str() around response.json['message'] because the unprocessable_entity helper nests the error under a 'json' key. --- .../tests/test_sensor_schedules_fresh_db.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index 601237dc6e..57b0cf91fe 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -303,7 +303,7 @@ def test_trigger_schedule_uses_state_of_charge_sensor_for_soc_at_start( @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_conflicting_consumption_is_positive_attribute_prevents_data_save( +def test_conflicting_consumption_is_positive_attribute_prevents_job_creation( app, fresh_db, add_market_prices_fresh_db, @@ -313,12 +313,13 @@ def test_conflicting_consumption_is_positive_attribute_prevents_data_save( keep_scheduling_queue_empty, requesting_user, ): - """Scheduling must fail before persisting any data when consumption_is_positive is already - set to a value that conflicts with the flex-model field used for the sensor. + """Scheduling must be rejected immediately when consumption_is_positive is already set + to a value that conflicts with the flex-model field used for the sensor. Here a sensor that already has ``consumption_is_positive=True`` (consumption sensor) is - mistakenly referenced under the ``production`` flex-model field. The scheduler should raise a - ``ValueError`` and leave the sensor's belief table empty. + mistakenly referenced under the ``production`` flex-model field. The trigger API should + raise a ``ValueError`` and return a 422 response, so the job is never created and the + sensor's belief table remains empty. ``force-new-job-creation`` is set to bypass the job-cache, which would otherwise serve the cached result of a prior test that ran against the same battery sensor and time window @@ -373,15 +374,12 @@ def test_conflicting_consumption_is_positive_attribute_prevents_data_save( url_for("SensorAPI:trigger_schedule", id=power_sensor.id), json=message, ) - assert trigger_response.status_code == 200 - job_id = trigger_response.json["schedule"] - - work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) - - # The job should have failed due to the attribute conflict. - job = Job.fetch(job_id, connection=app.queues["scheduling"].connection) - assert job.is_failed, "Expected the scheduling job to fail on attribute conflict" - assert "consumption_is_positive" in str(job.meta.get("exception", "")) + # The conflict is detected at trigger time: the API returns 422 immediately, + # without creating a job. + assert trigger_response.status_code == 422 + assert "consumption_is_positive" in str( + trigger_response.json.get("message", "") + ) # No beliefs should have been written for the mismatched sensor. mismatched_sensor = fresh_db.session.get(Sensor, mismatched_sensor.id) @@ -390,7 +388,7 @@ def test_conflicting_consumption_is_positive_attribute_prevents_data_save( ) assert ( len(saved_beliefs) == 0 - ), "No data should be saved when the attribute conflict is detected before save_to_db" + ), "No data should be saved when the attribute conflict is detected at job-creation time" @pytest.mark.parametrize( From 36d685d7759c9d840318fcd4e1b187e412bd9e81 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 28 May 2026 13:22:43 +0200 Subject: [PATCH 41/44] style: punctuation Signed-off-by: F.N. Claessen --- .../v3_0/tests/test_sensor_schedules_fresh_db.py | 8 ++++---- flexmeasures/data/models/planning/storage.py | 8 ++++---- flexmeasures/data/services/scheduling.py | 14 +++++++------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py index 57b0cf91fe..4ee841a502 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules_fresh_db.py @@ -317,7 +317,7 @@ def test_conflicting_consumption_is_positive_attribute_prevents_job_creation( to a value that conflicts with the flex-model field used for the sensor. Here a sensor that already has ``consumption_is_positive=True`` (consumption sensor) is - mistakenly referenced under the ``production`` flex-model field. The trigger API should + mistakenly referenced under the ``production`` flex-model field. The trigger API should raise a ``ValueError`` and return a 422 response, so the job is never created and the sensor's belief table remains empty. @@ -817,9 +817,9 @@ def test_get_schedule_sign_convention_json_flex_model( when the schedule was triggered with a consumption/production sensor in the JSON flex-model. The main power sensor is used as a reference (always fetched with the default - ``consumption-positive`` convention). The expected sign of the output sensor values - relative to that reference depends on ``output_type`` × ``sign_convention``; see - :func:`_assert_schedule_sign_convention` for the full decision table. + ``consumption-positive`` convention). The expected sign of the output sensor values + relative to that reference depends on ``output_type`` × ``sign_convention``; + see :func:`_assert_schedule_sign_convention` for the full decision table. """ battery_asset = add_battery_assets_fresh_db["Test battery"] diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index db8395c1b5..4af89ae816 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1601,19 +1601,19 @@ def _build_consumption_production_schedules( - **Only** ``consumption`` **sensor defined**: the full power schedule is written to that sensor using the scheduler's native sign convention (consumption positive, production - negative). ``make_schedule`` applies no further sign change because the sensor already + negative). ``make_schedule`` applies no further sign change because the sensor already has ``consumption_is_positive=True``. - **Only** ``production`` **sensor defined**: the full power schedule is written to that sensor in the scheduler's native sign convention (consumption positive, production - negative). ``make_schedule`` inverts the sign based on the sensor's + negative). ``make_schedule`` inverts the sign based on the sensor's ``consumption_is_positive=False`` attribute so that production is stored as positive values. - **Both** ``consumption`` **and** ``production`` **sensors defined**: only the non-negative part of the schedule (charging / consuming) is written to the consumption sensor, and only the non-positive part (discharging / producing, still as negative values) is written to - the production sensor. ``make_schedule`` inverts the sign for the production sensor. + the production sensor. ``make_schedule`` inverts the sign for the production sensor. The ``consumption_is_positive`` attribute is set on each output sensor when the scheduling - job is created (see ``create_scheduling_job``), not here. This method only clips the + job is created (see ``create_scheduling_job``), not here. This method only clips the series; sign handling is left entirely to ``make_schedule``. Unit conversion from MW to each sensor's unit is applied. diff --git a/flexmeasures/data/services/scheduling.py b/flexmeasures/data/services/scheduling.py index 772f51e388..5c7c7886fd 100644 --- a/flexmeasures/data/services/scheduling.py +++ b/flexmeasures/data/services/scheduling.py @@ -565,11 +565,11 @@ def _set_flex_model_output_sensors_consumption_is_positive( production sensor → consumption_is_positive = False A ``ValueError`` is raised immediately when the attribute is already present on a sensor - but has the wrong value for the flex-model field that references it. Calling this function + but has the wrong value for the flex-model field that references it. Calling this function at job-creation time lets the API surface conflicts before any work is queued. :param flex_model: Deserialized flex model — either a single-device ``dict`` or a - ``list`` of per-device dicts. Consumption/production fields are + ``list`` of per-device dicts. Consumption/production fields are expected to be dicts with a ``"sensor"`` key. :raises ValueError: When ``consumption_is_positive`` is already set to the wrong value for the given flex-model field. @@ -612,14 +612,14 @@ def _set_output_sensor_consumption_is_positive( """Set the ``consumption_is_positive`` attribute on a dedicated output sensor. For consumption output sensors the attribute is set to ``True`` (consumption is stored as - positive values). For production output sensors it is set to ``False`` (production is stored + positive values). For production output sensors it is set to ``False`` (production is stored as positive values, consumption as negative). The function is a no-op when *result* is not a dedicated consumption/production output schedule (as determined by :func:`_is_consumption_production_output`). A ``ValueError`` is raised when the attribute is already present on the sensor but points - in the wrong direction for the flex-model field being used. This check runs *before* any + in the wrong direction for the flex-model field being used. This check runs *before* any data are written so that the error surfaces as early as possible. :param result: Schedule output result dict with keys ``'name'``, ``'sensor'``, @@ -660,14 +660,14 @@ def _resolve_schedule_output_sign( is a power schedule that needs sign conversion so that production is stored as positive values in the database. - The scheduler always produces consumption-positive values. For sensors that carry + The scheduler always produces consumption-positive values. For sensors that carry ``consumption_is_positive=True`` (including consumption output sensors) no conversion - is needed. For sensors with ``consumption_is_positive=False`` (production output sensors + is needed. For sensors with ``consumption_is_positive=False`` (production output sensors and the default convention for main power sensors) the sign is inverted. .. note:: For consumption/production output sensors the ``consumption_is_positive`` attribute - must be set before this function is called. It is set eagerly at job-creation time + must be set before this function is called. It is set eagerly at job-creation time by :func:`_set_flex_model_output_sensors_consumption_is_positive`, and again (as a safety-net for direct :func:`make_schedule` calls) by :func:`_set_output_sensor_consumption_is_positive` earlier in the same loop iteration. From 2a7ca41f8a0f252b13b4f92e61d555223866e8b4 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 29 May 2026 18:05:05 +0200 Subject: [PATCH 42/44] docs: clarify effect of setting consumption_is_positive Signed-off-by: F.N. Claessen --- documentation/concepts/data-model.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index 94a5ae2f48..1aedfeb563 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -175,8 +175,10 @@ Note that, if forecasts are created, they will have the same sign as the origina For schedules, the sign of the power schedule (as :ref:`beliefs `) recorded in the database, and as seen in UI charts, is determined as follows: - If the flex-model contains the ``sensor`` field, and that sensor has power units (e.g. kW), the ``"consumption_is_positive"`` attribute of the sensor is used to decide the sign of the recorded data. - If the attribute is not defined, **by default, scheduled power is recorded with production as positive values** (and consumption as negative values). - To record scheduled power data with consumption as positive values, set ``sensor.attributes["consumption_is_positive"] = True``. + If `True`, consumption will be saved as positive, otherwise not. To clarify: + - If the attribute is not defined, **by default, scheduled power is recorded with production as positive values** (and consumption as negative values). + - To record scheduled power data with consumption as positive values, set ``sensor.attributes["consumption_is_positive"] = True``. + - To record scheduled power data with production as positive values (already the default, but this makes it explicit), set ``sensor.attributes["consumption_is_positive"] = False``. - If the flex-model contains the ``consumption`` field, scheduled power is recorded with consumption as positive values. The ``"consumption_is_positive"`` attribute of the referenced sensor is set automatically to ``True``. - If the flex-model contains the ``production`` field, scheduled power is recorded with production as positive values. From c5f51246499f75437977d02f6d3763b9b414a4c3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 29 May 2026 18:19:28 +0200 Subject: [PATCH 43/44] fix: improve test quality Signed-off-by: F.N. Claessen --- .../cli/tests/test_data_add_fresh_db.py | 99 ++++++++++++++++--- 1 file changed, 86 insertions(+), 13 deletions(-) diff --git a/flexmeasures/cli/tests/test_data_add_fresh_db.py b/flexmeasures/cli/tests/test_data_add_fresh_db.py index 66c81e8f9c..cfd50741b9 100644 --- a/flexmeasures/cli/tests/test_data_add_fresh_db.py +++ b/flexmeasures/cli/tests/test_data_add_fresh_db.py @@ -472,12 +472,41 @@ def test_add_storage_schedule_uses_state_of_charge_sensor_for_soc_at_start( add_charging_station_assets_fresh_db, setup_sources_fresh_db, ): + """Test that the StorageScheduler reads soc-at-start from a sensor and stores + schedules on dedicated consumption and production output sensors with the correct + sign convention and clipping behaviour. + + Setup: + - Bidirectional charging station (can both charge and discharge). + - SOC at start: 2.5 MWh (read from a sensor belief). + - Schedule window: 2015-01-03 00:00–12:00 CET. + On this date the market data has consumption prices of -10 EUR/MWh for hours 0–7 + (incentivises charging) and production prices of +60 EUR/MWh for hours 8–23 + (incentivises discharging), so the optimizer is guaranteed to do both. + + Sign-convention and clipping assertions (both consumption and production sensors defined): + - Consumption sensor (consumption_is_positive=True): all stored values ≥ 0 + (charging intervals are positive; discharging intervals are clipped to 0). + - Production sensor (consumption_is_positive=False): all stored values ≥ 0 + (discharging intervals are stored as positive; charging intervals are clipped to 0). + - No single timestep may carry both a positive consumption and a positive production + value (the two sensors together partition the schedule without overlap). + - At least one charging and one discharging event must actually occur. + """ from flexmeasures.cli.data_add import add_schedule - charging_station = add_charging_station_assets_fresh_db["Test charging station"] - power_sensor = next(s for s in charging_station.sensors if s.name == "power") - soc_sensor = add_charging_station_assets_fresh_db["uni-soc"] - start = "2015-01-01T00:00:00+01:00" + # Use the bidirectional station so both charging and discharging can occur. + bidirectional_charging_station = add_charging_station_assets_fresh_db[ + "Test charging station (bidirectional)" + ] + power_sensor = next( + s for s in bidirectional_charging_station.sensors if s.name == "power" + ) + soc_sensor = add_charging_station_assets_fresh_db["bi-soc"] + + # 2015-01-03: consumption prices are -10 EUR/MWh in hours 0-7 (charging rewarded) + # and production prices are +60 EUR/MWh in hours 8-23 (discharging rewarded). + start = "2015-01-03T00:00:00+01:00" fresh_db.session.add( TimedBelief( @@ -489,25 +518,37 @@ def test_add_storage_schedule_uses_state_of_charge_sensor_for_soc_at_start( ) ) - # Add a consumption output sensor to verify the full power profile is stored on it - # (only the consumption sensor is defined, so the sign convention is consumption positive, - # production negative). + # Add dedicated output sensors for the consumption (charging) and production + # (discharging) parts of the schedule. consumption_output_sensor = Sensor( name="consumption output", - generic_asset=charging_station, + generic_asset=bidirectional_charging_station, + unit="MW", + event_resolution=power_sensor.event_resolution, + ) + production_output_sensor = Sensor( + name="production output", + generic_asset=bidirectional_charging_station, unit="MW", event_resolution=power_sensor.event_resolution, ) fresh_db.session.add(consumption_output_sensor) + fresh_db.session.add(production_output_sensor) fresh_db.session.commit() + epex_da = add_market_prices_fresh_db["epex_da"] + epex_da_production = add_market_prices_fresh_db["epex_da_production"] + cli_input_params = { "sensor": power_sensor.id, "start": start, "duration": "PT12H", "scheduler": "StorageScheduler", "flex-context": json.dumps( - {"consumption-price": {"sensor": add_market_prices_fresh_db["epex_da"].id}} + { + "consumption-price": {"sensor": epex_da.id}, + "production-price": {"sensor": epex_da_production.id}, + } ), "flex-model": json.dumps( { @@ -516,6 +557,7 @@ def test_add_storage_schedule_uses_state_of_charge_sensor_for_soc_at_start( "soc-max": "5 MWh", "power-capacity": "2 MW", "consumption": {"sensor": consumption_output_sensor.id}, + "production": {"sensor": production_output_sensor.id}, } ), } @@ -526,10 +568,41 @@ def test_add_storage_schedule_uses_state_of_charge_sensor_for_soc_at_start( assert len(power_sensor.search_beliefs()) == 48 assert power_sensor.generic_asset.get_attribute("soc_in_mwh") == 2.5 - # Verify the consumption output sensor received the full power schedule. - # A charging station is consumption-only (non-negative), so the full schedule - # is non-negative and equals what is stored on the power sensor. + # Reload sensors from the DB after the schedule has been committed. consumption_output_sensor = fresh_db.session.get( Sensor, consumption_output_sensor.id ) - assert len(consumption_output_sensor.search_beliefs()) == 48 + production_output_sensor = fresh_db.session.get(Sensor, production_output_sensor.id) + consumption_beliefs = consumption_output_sensor.search_beliefs() + production_beliefs = production_output_sensor.search_beliefs() + + assert len(consumption_beliefs) == 48 + assert len(production_beliefs) == 48 + + consumption_values = consumption_beliefs.values.flatten() + production_values = production_beliefs.values.flatten() + + # Sign convention: consumption sensor (consumption_is_positive=True) stores + # charging as positive values; discharging intervals are clipped to 0. + assert ( + consumption_values >= 0 + ).all(), "Consumption output sensor must only hold non-negative values" + # Sign convention: production sensor (consumption_is_positive=False) stores + # discharging as positive values; charging intervals are clipped to 0. + assert ( + production_values >= 0 + ).all(), "Production output sensor must only hold non-negative values" + # Clipping: the two sensors partition the schedule without overlap — no + # single timestep should carry both a positive consumption and a positive + # production value. + assert not ( + (consumption_values > 0) & (production_values > 0) + ).any(), "No timestep may have both positive consumption and positive production" + # With negative consumption prices in hours 0-7 and positive production prices + # in hours 8+, both charging and discharging must occur. + assert ( + consumption_values > 0 + ).any(), "Some charging must occur given the negative consumption prices" + assert ( + production_values > 0 + ).any(), "Some discharging must occur given the positive production prices" From 5ea784c99049e2ffc964bb448d5075a9115c4093 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 30 May 2026 11:17:02 +0200 Subject: [PATCH 44/44] docs: use past tense to discriminate between what the endpoint does and what the scheduler did Signed-off-by: F.N. Claessen --- documentation/api/notation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index 969ff22f92..68b98e9d3a 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -308,4 +308,4 @@ The ``GET /api/v3_0/sensors//schedules/`` endpoint supports three sign - ``consumption-positive`` (**default**): schedules are returned with consumption as positive values and production as negative values, regardless of how they are stored in the database. - ``production-positive``: schedules are returned with production as positive values and consumption as negative values. - ``wysiwyg`` (*what-you-see-is-what-you-get*): schedules are returned with the same sign as database values and as seen in the UI charts. - The values indicate exactly what is stored, which is itself governed by the sensor's ``consumption_is_positive`` attribute (if present) or the scheduler's default convention (to store production as positive values in the database). + The values indicate exactly what was stored, which was itself governed by the sensor's ``consumption_is_positive`` attribute (if present) or the scheduler's default convention (which stored production as positive values in the database).