From 129cba22746b0f4e9e7c923f3bf59ba2b16fb2f2 Mon Sep 17 00:00:00 2001 From: filimarc Date: Fri, 16 Jan 2026 14:59:28 +0100 Subject: [PATCH 01/18] fix: make flush method to save in file --- packages/bsb-arbor/bsb_arbor/adapter.py | 16 ++++---- .../bsb-core/bsb/cli/commands/_commands.py | 2 +- packages/bsb-core/bsb/core.py | 4 +- packages/bsb-core/bsb/simulation/adapter.py | 10 ++--- packages/bsb-core/bsb/simulation/results.py | 40 ++++++++++++------- packages/bsb-nest/bsb_nest/adapter.py | 8 ++-- packages/bsb-neuron/bsb_neuron/adapter.py | 13 ++++-- 7 files changed, 57 insertions(+), 36 deletions(-) diff --git a/packages/bsb-arbor/bsb_arbor/adapter.py b/packages/bsb-arbor/bsb_arbor/adapter.py index 72d874ef..1d5480a2 100644 --- a/packages/bsb-arbor/bsb_arbor/adapter.py +++ b/packages/bsb-arbor/bsb_arbor/adapter.py @@ -25,11 +25,11 @@ class ArborSimulationData(SimulationData): Container class for simulation data. """ - def __init__(self, simulation): + def __init__(self, simulation, filename): """ Container class for simulation data. """ - super().__init__(simulation) + super().__init__(simulation, filename) self.arbor_sim: arbor.simulation = None @@ -234,7 +234,9 @@ def __iter__(self): :yield: Each GID in the population's ranges """ - yield from itertools.chain.from_iterable(range(r[0], r[1]) for r in self._ranges) + yield from itertools.chain.from_iterable( + range(r[0], r[1]) for r in self._ranges + ) class GIDManager: @@ -375,11 +377,11 @@ def __init__(self, comm=None): super().__init__(comm) self.simdata: dict[ArborSimulation, ArborSimulationData] = {} - def prepare(self, simulation: "ArborSimulation") -> ArborSimulationData: + def prepare(self, simulation: "ArborSimulation", filename) -> ArborSimulationData: """ Prepares the arbor simulation engine with the given simulation. """ - simdata = self._create_simdata(simulation) + simdata = self._create_simdata(simulation, filename) try: context = arbor.context(arbor.proc_allocation(threads=simulation.threads)) if self.comm.get_size() > 1: @@ -466,8 +468,8 @@ def get_recipe(self, simulation, simdata=None): self._cache_devices(simulation, simdata) return ArborRecipe(simulation, simdata) - def _create_simdata(self, simulation): - self.simdata[simulation] = simdata = ArborSimulationData(simulation) + def _create_simdata(self, simulation, filename): + self.simdata[simulation] = simdata = ArborSimulationData(simulation, filename) self._assign_chunks(simulation, simdata) return simdata diff --git a/packages/bsb-core/bsb/cli/commands/_commands.py b/packages/bsb-core/bsb/cli/commands/_commands.py index cc9c402a..345c5e28 100644 --- a/packages/bsb-core/bsb/cli/commands/_commands.py +++ b/packages/bsb-core/bsb/cli/commands/_commands.py @@ -229,7 +229,7 @@ def handler(self, context): level=0, ) try: - result = network.run_simulation(sim_name) + result = network.run_simulation(sim_name, root / f"{uuid4()}.nio") except NodeNotFoundError as e: append = ", " if len(network.simulations) else "" append += ", ".join(f"'{name}'" for name in extra_simulations) diff --git a/packages/bsb-core/bsb/core.py b/packages/bsb-core/bsb/core.py index 9a1125f8..caf8ec25 100644 --- a/packages/bsb-core/bsb/core.py +++ b/packages/bsb-core/bsb/core.py @@ -449,7 +449,7 @@ def run_pipelines(self, fail_fast=True, pipelines=None): pool.schedule(pipelines) pool.execute() - def run_simulation(self, simulation_name: str): + def run_simulation(self, simulation_name: str, output_filename: str = None): """ Run a simulation starting from the default single-instance adapter. @@ -460,7 +460,7 @@ def run_simulation(self, simulation_name: str): adapter = get_simulation_adapter( simulation.simulator, comm=self._comm.get_communicator() ) - return adapter.simulate(simulation)[0] + return adapter.simulate(simulation, filename=output_filename)[0] def get_simulation(self, sim_name: str) -> Simulation: """ diff --git a/packages/bsb-core/bsb/simulation/adapter.py b/packages/bsb-core/bsb/simulation/adapter.py index b31e28a6..1815d756 100644 --- a/packages/bsb-core/bsb/simulation/adapter.py +++ b/packages/bsb-core/bsb/simulation/adapter.py @@ -66,7 +66,7 @@ def use_bar(self): class SimulationData: - def __init__(self, simulation: "Simulation", result=None): + def __init__(self, simulation: "Simulation", result=None, filename=None): self.chunks = None self.populations = dict() self.placement: dict[CellModel, PlacementSet] = { @@ -75,7 +75,7 @@ def __init__(self, simulation: "Simulation", result=None): self.connections = dict() self.devices = dict() if result is None: - result = SimulationResult(simulation) + result = SimulationResult(simulation, filename=filename) self.result: SimulationResult = result @@ -92,7 +92,7 @@ def __init__(self, comm=None): self._duration = None self.current_checkpoint = 0 - def simulate(self, *simulations, post_prepare=None): + def simulate(self, *simulations, post_prepare=None, filename=None): """ Simulate the given simulations. @@ -113,7 +113,7 @@ def simulate(self, *simulations, post_prepare=None): self._controllers.append(listener) for simulation in simulations: - data = self.prepare(simulation) + data = self.prepare(simulation, filename) alldata.append(data) for hook in simulation.post_prepare: hook(self, simulation, data) @@ -123,7 +123,7 @@ def simulate(self, *simulations, post_prepare=None): return self.collect(results) @abc.abstractmethod - def prepare(self, simulation): # pragma: nocover + def prepare(self, simulation, filename): # pragma: nocover """ Reset the simulation backend and prepare for the given simulation. diff --git a/packages/bsb-core/bsb/simulation/results.py b/packages/bsb-core/bsb/simulation/results.py index 1547d22e..f5422902 100644 --- a/packages/bsb-core/bsb/simulation/results.py +++ b/packages/bsb-core/bsb/simulation/results.py @@ -9,22 +9,27 @@ class SimulationResult: - def __init__(self, simulation): - from neo import Block + def __init__(self, simulation, filename=None): + from neo import Block, io tree = simulation.__tree__() with contextlib.suppress(KeyError): del tree["post_prepare"] - self.block = Block(name=simulation.name, config=tree) - self.recorders = [] - - @property - def spiketrains(self): - return self.block.segments[0].spiketrains + if filename: + self.filename = filename + self.name = simulation.name + io = io.NixIO(filename, mode="rw") + io.write(Block(name=self.name, nix_name=self.name, config=tree)) + for i, nixblock in enumerate(io.nix_file.blocks): + if self.name == nixblock.name: + self.block_id = i + io.close() + else: + self.block = Block( + name=simulation.name, nix_name=simulation.name, config=tree + ) - @property - def analogsignals(self): - return self.block.segments[0].analogsignals + self.recorders = [] def add(self, recorder): self.recorders.append(recorder) @@ -36,21 +41,28 @@ def create_recorder(self, flush: typing.Callable[["neo.core.Segment"], None]): return recorder def flush(self): - from neo import Segment + from neo import Segment, io segment = Segment() - self.block.segments.append(segment) for recorder in self.recorders: try: recorder.flush(segment) except Exception: traceback.print_exc() warn("Recorder errored out!") + if hasattr(self, "filename"): + io = io.NixIO(self.filename, mode="rw") + block = io.nix_file.blocks[self.block_id] + io._write_segment(segment, block) + io.close() + else: + self.block.segments.append(segment) def write(self, filename, mode): from neo import io - io.NixIO(filename, mode=mode).write(self.block) + if hasattr(self, "block"): + io.NixIO(filename, mode=mode).write(self.block) class SimulationRecorder: diff --git a/packages/bsb-nest/bsb_nest/adapter.py b/packages/bsb-nest/bsb_nest/adapter.py index 7accd0ca..aeb16bf3 100644 --- a/packages/bsb-nest/bsb_nest/adapter.py +++ b/packages/bsb-nest/bsb_nest/adapter.py @@ -58,7 +58,7 @@ def simulate(self, *simulations, post_prepare=None): finally: self.reset_kernel() - def prepare(self, simulation): + def prepare(self, simulation, filename): """ Prepare the simulation environment in NEST. @@ -80,7 +80,7 @@ def prepare(self, simulation): :rtype: bsb.simulation.adapter.SimulationData """ self.simdata[simulation] = SimulationData( - simulation, result=NestResult(simulation) + simulation, result=NestResult(simulation, filename) ) try: report("Installing NEST modules...", level=2) @@ -186,7 +186,9 @@ def connect_neurons(self, simulation): ) ) except Exception as e: - raise NestConnectError(f"{connection_model} error during connect.") from e + raise NestConnectError( + f"{connection_model} error during connect." + ) from e def set_settings(self, simulation: "NestSimulation"): nest.set_verbosity(simulation.verbosity) diff --git a/packages/bsb-neuron/bsb_neuron/adapter.py b/packages/bsb-neuron/bsb_neuron/adapter.py index d00ec34f..3ddde80f 100644 --- a/packages/bsb-neuron/bsb_neuron/adapter.py +++ b/packages/bsb-neuron/bsb_neuron/adapter.py @@ -70,7 +70,7 @@ def engine(self): return engine - def prepare(self, simulation): + def prepare(self, simulation, filename): """ Prepare the simulation environment and data structures for running a NEURON simulation. @@ -90,7 +90,8 @@ def prepare(self, simulation): """ self.simdata[simulation] = NeuronSimulationData( - simulation, result=NeuronResult(simulation) + simulation, + result=NeuronResult(simulation, filename=filename), ) try: report("Preparing simulation", level=2) @@ -192,7 +193,9 @@ def _map_transceivers(self, simulation, simdata): offset = 0 transmap = {} - pre_types = set(cs.pre_type for cs in simulation.get_connectivity_sets().values()) + pre_types = set( + cs.pre_type for cs in simulation.get_connectivity_sets().values() + ) for pre_type in sorted(pre_types, key=lambda pre_type: pre_type.name): data = [] for _cm, cs in simulation.get_connectivity_sets().items(): @@ -209,7 +212,9 @@ def _map_transceivers(self, simulation, simdata): continue # Now look up which transmitters are on our chunks - pre_t, _ = cs.load_connections().from_(simdata.chunks).as_globals().all() + pre_t, _ = ( + cs.load_connections().from_(simdata.chunks).as_globals().all() + ) our_cm_transmitters = np.unique(pre_t[:, :2], axis=0) # Look up the local ids of those transmitters pre_lc, _ = cs.load_connections().from_(simdata.chunks).all() From 98f2e066431f1adbe60ebb3ec0521619bc240d64 Mon Sep 17 00:00:00 2001 From: filimarc Date: Mon, 19 Jan 2026 11:59:34 +0100 Subject: [PATCH 02/18] fix: filename options and sim tests --- examples/nest-simulation/tests/test_examples.py | 4 ++-- examples/neuron-simulation/tests/test_examples.py | 4 ++-- packages/bsb-arbor/bsb_arbor/adapter.py | 8 ++++---- packages/bsb-core/bsb/simulation/adapter.py | 2 +- packages/bsb-nest/bsb_nest/adapter.py | 12 ++++++------ packages/bsb-neuron/bsb_neuron/adapter.py | 10 +++------- 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/examples/nest-simulation/tests/test_examples.py b/examples/nest-simulation/tests/test_examples.py index f7c65953..28fe210f 100644 --- a/examples/nest-simulation/tests/test_examples.py +++ b/examples/nest-simulation/tests/test_examples.py @@ -62,7 +62,7 @@ def test_json_example(self): self.scaffold.compile() self._test_scaffold_results() results = self.scaffold.run_simulation("basal_activity") - self._test_simulation_results(results.spiketrains) + self._test_simulation_results(results.block.segments[0].spiketrains) def test_yaml_example(self): self.cfg = parse_configuration_file( @@ -72,7 +72,7 @@ def test_yaml_example(self): self.scaffold.compile() self._test_scaffold_results() results = self.scaffold.run_simulation("basal_activity") - self._test_simulation_results(results.spiketrains) + self._test_simulation_results(results.block.segments[0].spiketrains) def test_python_example(self): import scripts.guide_nest # noqa: F401 diff --git a/examples/neuron-simulation/tests/test_examples.py b/examples/neuron-simulation/tests/test_examples.py index 8ecd2208..0cefa8d1 100644 --- a/examples/neuron-simulation/tests/test_examples.py +++ b/examples/neuron-simulation/tests/test_examples.py @@ -67,7 +67,7 @@ def test_json_example(self): self.scaffold.compile() self._test_scaffold_results() results = self.scaffold.run_simulation("neuronsim") - self._test_simulation_results(results.analogsignals) + self._test_simulation_results(results.block.segments[0].analogsignals) def test_yaml_example(self): self.cfg = parse_configuration_file( @@ -77,7 +77,7 @@ def test_yaml_example(self): self.scaffold.compile() self._test_scaffold_results() results = self.scaffold.run_simulation("neuronsim") - self._test_simulation_results(results.analogsignals) + self._test_simulation_results(results.block.segments[0].analogsignals) def test_python_example(self): import scripts.guide_neuron # noqa: F401 diff --git a/packages/bsb-arbor/bsb_arbor/adapter.py b/packages/bsb-arbor/bsb_arbor/adapter.py index 1d5480a2..db77c602 100644 --- a/packages/bsb-arbor/bsb_arbor/adapter.py +++ b/packages/bsb-arbor/bsb_arbor/adapter.py @@ -234,9 +234,7 @@ def __iter__(self): :yield: Each GID in the population's ranges """ - yield from itertools.chain.from_iterable( - range(r[0], r[1]) for r in self._ranges - ) + yield from itertools.chain.from_iterable(range(r[0], r[1]) for r in self._ranges) class GIDManager: @@ -377,7 +375,9 @@ def __init__(self, comm=None): super().__init__(comm) self.simdata: dict[ArborSimulation, ArborSimulationData] = {} - def prepare(self, simulation: "ArborSimulation", filename) -> ArborSimulationData: + def prepare( + self, simulation: "ArborSimulation", filename=None + ) -> ArborSimulationData: """ Prepares the arbor simulation engine with the given simulation. """ diff --git a/packages/bsb-core/bsb/simulation/adapter.py b/packages/bsb-core/bsb/simulation/adapter.py index 1815d756..b365c303 100644 --- a/packages/bsb-core/bsb/simulation/adapter.py +++ b/packages/bsb-core/bsb/simulation/adapter.py @@ -123,7 +123,7 @@ def simulate(self, *simulations, post_prepare=None, filename=None): return self.collect(results) @abc.abstractmethod - def prepare(self, simulation, filename): # pragma: nocover + def prepare(self, simulation, filename=None): # pragma: nocover """ Reset the simulation backend and prepare for the given simulation. diff --git a/packages/bsb-nest/bsb_nest/adapter.py b/packages/bsb-nest/bsb_nest/adapter.py index aeb16bf3..13109a13 100644 --- a/packages/bsb-nest/bsb_nest/adapter.py +++ b/packages/bsb-nest/bsb_nest/adapter.py @@ -51,14 +51,16 @@ def __init__(self, comm=None): self.loaded_modules = set() self._prev_chkpoint = 0 - def simulate(self, *simulations, post_prepare=None): + def simulate(self, *simulations, post_prepare=None, filename=None): try: self.reset_kernel() - return super().simulate(*simulations, post_prepare=post_prepare) + return super().simulate( + *simulations, post_prepare=post_prepare, filename=filename + ) finally: self.reset_kernel() - def prepare(self, simulation, filename): + def prepare(self, simulation, filename=None): """ Prepare the simulation environment in NEST. @@ -186,9 +188,7 @@ def connect_neurons(self, simulation): ) ) except Exception as e: - raise NestConnectError( - f"{connection_model} error during connect." - ) from e + raise NestConnectError(f"{connection_model} error during connect.") from e def set_settings(self, simulation: "NestSimulation"): nest.set_verbosity(simulation.verbosity) diff --git a/packages/bsb-neuron/bsb_neuron/adapter.py b/packages/bsb-neuron/bsb_neuron/adapter.py index 3ddde80f..8402ca50 100644 --- a/packages/bsb-neuron/bsb_neuron/adapter.py +++ b/packages/bsb-neuron/bsb_neuron/adapter.py @@ -70,7 +70,7 @@ def engine(self): return engine - def prepare(self, simulation, filename): + def prepare(self, simulation, filename=None): """ Prepare the simulation environment and data structures for running a NEURON simulation. @@ -193,9 +193,7 @@ def _map_transceivers(self, simulation, simdata): offset = 0 transmap = {} - pre_types = set( - cs.pre_type for cs in simulation.get_connectivity_sets().values() - ) + pre_types = set(cs.pre_type for cs in simulation.get_connectivity_sets().values()) for pre_type in sorted(pre_types, key=lambda pre_type: pre_type.name): data = [] for _cm, cs in simulation.get_connectivity_sets().items(): @@ -212,9 +210,7 @@ def _map_transceivers(self, simulation, simdata): continue # Now look up which transmitters are on our chunks - pre_t, _ = ( - cs.load_connections().from_(simdata.chunks).as_globals().all() - ) + pre_t, _ = cs.load_connections().from_(simdata.chunks).as_globals().all() our_cm_transmitters = np.unique(pre_t[:, :2], axis=0) # Look up the local ids of those transmitters pre_lc, _ = cs.load_connections().from_(simdata.chunks).all() From 3ae47466ea4bb6b881d1d1480640f785951c0bb7 Mon Sep 17 00:00:00 2001 From: filimarc Date: Tue, 20 Jan 2026 10:34:58 +0100 Subject: [PATCH 03/18] fix: backward compatibility --- packages/bsb-arbor/bsb_arbor/adapter.py | 2 +- packages/bsb-core/bsb/cli/commands/_commands.py | 6 +++--- packages/bsb-core/bsb/simulation/results.py | 10 ++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/bsb-arbor/bsb_arbor/adapter.py b/packages/bsb-arbor/bsb_arbor/adapter.py index db77c602..b64021cb 100644 --- a/packages/bsb-arbor/bsb_arbor/adapter.py +++ b/packages/bsb-arbor/bsb_arbor/adapter.py @@ -29,7 +29,7 @@ def __init__(self, simulation, filename): """ Container class for simulation data. """ - super().__init__(simulation, filename) + super().__init__(simulation, filename=filename) self.arbor_sim: arbor.simulation = None diff --git a/packages/bsb-core/bsb/cli/commands/_commands.py b/packages/bsb-core/bsb/cli/commands/_commands.py index 345c5e28..c3a6779f 100644 --- a/packages/bsb-core/bsb/cli/commands/_commands.py +++ b/packages/bsb-core/bsb/cli/commands/_commands.py @@ -229,13 +229,13 @@ def handler(self, context): level=0, ) try: - result = network.run_simulation(sim_name, root / f"{uuid4()}.nio") + network.run_simulation(sim_name, output_filename=root / f"{uuid4()}.nio") except NodeNotFoundError as e: append = ", " if len(network.simulations) else "" append += ", ".join(f"'{name}'" for name in extra_simulations) errr.wrap(type(e), e, append=append) - else: - result.write(root / f"{uuid4()}.nio", "ow") + # else: + # result.write(root / f"{uuid4()}.nio", "ow") def get_options(self): return { diff --git a/packages/bsb-core/bsb/simulation/results.py b/packages/bsb-core/bsb/simulation/results.py index f5422902..fbf68025 100644 --- a/packages/bsb-core/bsb/simulation/results.py +++ b/packages/bsb-core/bsb/simulation/results.py @@ -31,6 +31,16 @@ def __init__(self, simulation, filename=None): self.recorders = [] + @property + def analogsignals(self): + if hasattr(self, "block"): + return self.block.segments[0].analogsignals + + @property + def spiketrains(self): + if hasattr(self, "block"): + return self.block.segments[0].spiketrains + def add(self, recorder): self.recorders.append(recorder) From 895f70105ad2c551a9607d292c489817cd7b14ff Mon Sep 17 00:00:00 2001 From: Filippo <52816133+filimarc@users.noreply.github.com> Date: Wed, 25 Mar 2026 09:08:03 +0100 Subject: [PATCH 04/18] Update packages/bsb-core/bsb/cli/commands/_commands.py Co-authored-by: Robin De Schepper --- packages/bsb-core/bsb/cli/commands/_commands.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/bsb-core/bsb/cli/commands/_commands.py b/packages/bsb-core/bsb/cli/commands/_commands.py index c3a6779f..429ea238 100644 --- a/packages/bsb-core/bsb/cli/commands/_commands.py +++ b/packages/bsb-core/bsb/cli/commands/_commands.py @@ -234,8 +234,6 @@ def handler(self, context): append = ", " if len(network.simulations) else "" append += ", ".join(f"'{name}'" for name in extra_simulations) errr.wrap(type(e), e, append=append) - # else: - # result.write(root / f"{uuid4()}.nio", "ow") def get_options(self): return { From 304b83fdbb77fb9ca6a361d90bde89074696c818 Mon Sep 17 00:00:00 2001 From: filimarc Date: Mon, 13 Apr 2026 10:44:50 +0200 Subject: [PATCH 05/18] test: add double simulation test --- packages/bsb-core/bsb/simulation/results.py | 8 +-- packages/bsb-core/tests/test_simulation.py | 80 ++++++++++++++++++++- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/packages/bsb-core/bsb/simulation/results.py b/packages/bsb-core/bsb/simulation/results.py index fbf68025..2b7f095a 100644 --- a/packages/bsb-core/bsb/simulation/results.py +++ b/packages/bsb-core/bsb/simulation/results.py @@ -61,10 +61,10 @@ def flush(self): traceback.print_exc() warn("Recorder errored out!") if hasattr(self, "filename"): - io = io.NixIO(self.filename, mode="rw") - block = io.nix_file.blocks[self.block_id] - io._write_segment(segment, block) - io.close() + out_stream = io.NixIO(self.filename, mode="rw") + block = out_stream.nix_file.blocks[self.block_id] + out_stream._write_segment(segment, block) + out_stream.close() else: self.block.segments.append(segment) diff --git a/packages/bsb-core/tests/test_simulation.py b/packages/bsb-core/tests/test_simulation.py index 7ce18690..66355613 100644 --- a/packages/bsb-core/tests/test_simulation.py +++ b/packages/bsb-core/tests/test_simulation.py @@ -1,8 +1,9 @@ import unittest import numpy as np -from bsb_arbor import SpikeRecorder +from bsb_arbor import ArborSimulation, SpikeRecorder from bsb_test import FixedPosConfigFixture, NumpyTestCase, RandomStorageFixture +from neo import io from bsb import ( MPI, @@ -563,3 +564,80 @@ def test_two_controllers(self): & (segments[-1].spiketrains[0].magnitude < 100), "Spike times in last segment fall outside the expected range (90-100).", ) + + def test_checkpoints_with_double_sim(self): + """This test checks that if two simulations are run the results are written in two + separate blocks inside the same file""" + self.network.simulations.test.devices["new_recorder"] = dict( + device="spike_controller", + targetting={ + "strategy": "cell_model", + "cell_models": ["test_cell"], + }, + step=10, + ) + sim_2 = dict( + simulator="arbor", + duration=50, + resolution=0.5, + cell_models={ + "test_cell": { + "model_strategy": "lif", + "constants": { + "C_m": 250, + "tau_m": 20, + "t_ref": 2.0, + "E_L": 0.0, + "E_R": 0.0, + "V_m": 0.0, + "V_th": 20, + }, + }, + "h_cell": { + "model_strategy": "lif", + "constants": { + "C_m": 250, + "tau_m": 20, + "t_ref": 2.0, + "E_L": 0.0, + "E_R": 0.0, + "V_m": 0.0, + "V_th": 20, + }, + }, + }, + connection_models={ + "test_to_h_cell": {"weight": 20.68015524367846, "delay": 1.5} + }, + devices=dict( + pg={ + "device": "poisson_generator", + "rate": 1600, + "targetting": {"strategy": "all"}, + "weight": 2000, + "delay": 1.5, + }, + new_recorder=dict( + device="spike_controller", + targetting={ + "strategy": "cell_model", + "cell_models": ["test_cell"], + }, + step=10, + ), + ), + ) + self.network.simulations["sim_2"] = ArborSimulation(sim_2) + + nio_file = "out.nio" + self.network.run_simulation("test", output_filename=nio_file) + self.network.run_simulation("sim_2", output_filename=nio_file) + + written_results = io.NixIO(nio_file, "ro") + blocks = written_results.read_all_blocks() + self.assertEqual(len(blocks), 2) + self.assertEqual(len(blocks[0].segments), 11) + self.assertEqual(len(blocks[1].segments), 6) + import os + + os.remove(nio_file) From 3a527faa91f31394ea71154a19d0a9a0f6c324e6 Mon Sep 17 00:00:00 2001 From: filimarc Date: Tue, 14 Apr 2026 15:20:29 +0200 Subject: [PATCH 06/18] test: add test for RAM usage --- packages/bsb-neuron/tests/test_neuron.py | 128 ++++++++++++++++++++++- 1 file changed, 125 insertions(+), 3 deletions(-) diff --git a/packages/bsb-neuron/tests/test_neuron.py b/packages/bsb-neuron/tests/test_neuron.py index 03f7c87c..548fd40d 100644 --- a/packages/bsb-neuron/tests/test_neuron.py +++ b/packages/bsb-neuron/tests/test_neuron.py @@ -2,9 +2,7 @@ import unittest from copy import copy -from bsb.core import Scaffold -from bsb.services import MPI -from bsb.simulation import get_simulation_adapter +from bsb import MPI, Scaffold, config, get_simulation_adapter from bsb_test import ( ConfigFixture, MorphologiesFixture, @@ -15,6 +13,7 @@ from bsb_neuron.cell import ArborizedModel from bsb_neuron.connection import TransceiverModel +from bsb_neuron.devices import VoltageRecorder class TestNeuronMinimal( @@ -525,3 +524,126 @@ def test_500ch_manualloop(self): ], receiving_cells, ) + + +class TestCheckpoints( + RandomStorageFixture, + ConfigFixture, + NetworkFixture, + MorphologiesFixture, + unittest.TestCase, + config="complete", + morpho_filters=["3branch"], + engine_name="hdf5", +): + def setUp(self): + import os + + import psutil + + super().setUp() + p.parallel.gid_clear() + for ct in self.network.cell_types.values(): + ct.spatial.morphologies = ["3branch"] + + hh_soma = { + "cable_types": { + "soma": { + "cable": {"Ra": 10, "cm": 1}, + "mechanisms": {"pas": {}, "hh": {}}, + } + }, + "synapse_types": {"ExpSyn": {}}, + } + devices = { + "spike_generator": { + "device": "spike_generator", + "start": 9, + "number": 8, + "weight": 1, + "delay": 1, + "targetting": { + "strategy": "cell_model", + "cell_models": ["A", "B", "C"], + }, + } + } + + for i in range(200): + devices[str(i)] = { + "device": "voltage_recorder", + "targetting": {"strategy": "cell_model", "cell_models": ["A", "B", "C"]}, + } + + self.network.simulations.add( + "test", + simulator="neuron", + duration=10000, + resolution=0.1, + temperature=32, + cell_models=dict( + A=ArborizedModel(model=hh_soma), + B=ArborizedModel(model=hh_soma), + C=ArborizedModel(model=hh_soma), + ), + connection_models=dict( + A_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), + A_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), + B_to_C=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), + C_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), + C_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), + ), + devices=devices, + ) + self.network.compile() + print(f"{self.network.simulations.test.duration}") + self.process = psutil.Process(os.getpid()) + # Baseline before the test + self.before_mem = self.process.memory_info().rss + + def test_RAM_usage(self): + import psutil + + @config.node + class SpikeController( + VoltageRecorder, + classmap_entry="ram_controller", + ): + threshold = config.attr(type=float, required=True) + + def __init__(self, **kwargs): + super().__init__() + self._status = 1 + self._memory = psutil.virtual_memory() + + def implement(self, adapter, simulation, simdata): + super().implement(adapter, simulation, simdata) + self._simdata = simdata + + def get_next_checkpoint(self): + return self._status + + def run_checkpoint(self, kwargs=None): + # If threshold is reached Flush data + if self._memory.percent > self.threshold: + self._simdata.result.flush() + + self._status += 1 + + return self._status + + self.network.simulations.test.devices["new"] = dict( + device="spike_controller", + targetting={ + "strategy": "cell_model", + "cell_models": ["A", "B", "C"], + }, + threshold=75, + ) + self.network.run_simulation("test", "out.nio") + + def tearDown(self): + # Memory after the test + after_mem = self.process.memory_info().rss + delta = (after_mem - self.before_mem) / (1024**2) + print(f"\n{self._testMethodName} used {delta:.2f} MB") From 847ec0253aba7d27ada24b3ec6362371941ddf3e Mon Sep 17 00:00:00 2001 From: filimarc Date: Wed, 15 Apr 2026 15:09:41 +0200 Subject: [PATCH 07/18] fix: install-nest.sh --- devtools/install-nest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devtools/install-nest.sh b/devtools/install-nest.sh index 4207f02b..9068c35d 100755 --- a/devtools/install-nest.sh +++ b/devtools/install-nest.sh @@ -1,6 +1,6 @@ # Get Nest installation folder +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )" if [ -z "$NEST_FOLDER" ]; then - SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"; NEST_FOLDER="$(dirname $SCRIPT_DIR)/.nx/installation/nest"; fi # Get NEST version @@ -13,7 +13,7 @@ if [ -f "$INSTALLATION_FOLDER/bin/nest_vars.sh" ]; then fi # Lock check and installation to prevent concurrent file edition -LOCK_FILE="/tmp/bsb-nest.lock" +LOCK_FILE="$SCRIPT_DIR/bsb-nest.lock" # Remove lock file on exit trap 'rm -f "$LOCK_FILE"' EXIT # Wait to be able to create file From 34e878f59db45395a081cb25ea00f6fdbe6576e1 Mon Sep 17 00:00:00 2001 From: filimarc Date: Thu, 16 Apr 2026 16:41:17 +0200 Subject: [PATCH 08/18] test: fix test for RAM usage --- packages/bsb-core/bsb/simulation/results.py | 2 +- packages/bsb-neuron/tests/test_neuron.py | 40 +++++++++++---------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/packages/bsb-core/bsb/simulation/results.py b/packages/bsb-core/bsb/simulation/results.py index 2b7f095a..a36d6e5d 100644 --- a/packages/bsb-core/bsb/simulation/results.py +++ b/packages/bsb-core/bsb/simulation/results.py @@ -52,7 +52,6 @@ def create_recorder(self, flush: typing.Callable[["neo.core.Segment"], None]): def flush(self): from neo import Segment, io - segment = Segment() for recorder in self.recorders: try: @@ -65,6 +64,7 @@ def flush(self): block = out_stream.nix_file.blocks[self.block_id] out_stream._write_segment(segment, block) out_stream.close() + del segment else: self.block.segments.append(segment) diff --git a/packages/bsb-neuron/tests/test_neuron.py b/packages/bsb-neuron/tests/test_neuron.py index 548fd40d..00de2591 100644 --- a/packages/bsb-neuron/tests/test_neuron.py +++ b/packages/bsb-neuron/tests/test_neuron.py @@ -569,7 +569,7 @@ def setUp(self): } } - for i in range(200): + for i in range(20): devices[str(i)] = { "device": "voltage_recorder", "targetting": {"strategy": "cell_model", "cell_models": ["A", "B", "C"]}, @@ -578,7 +578,7 @@ def setUp(self): self.network.simulations.add( "test", simulator="neuron", - duration=10000, + duration=20000, resolution=0.1, temperature=32, cell_models=dict( @@ -602,34 +602,37 @@ def setUp(self): self.before_mem = self.process.memory_info().rss def test_RAM_usage(self): - import psutil - + """This test run a simulation that records around 1.1 GB of data and check that + the maximum peak of memory do not exceed 600 MB and that after every flush + the usage is at least 250 MB below the peak.""" + import tracemalloc + tracemalloc.start() @config.node class SpikeController( VoltageRecorder, classmap_entry="ram_controller", ): - threshold = config.attr(type=float, required=True) + step = config.attr(type=float, required=True) def __init__(self, **kwargs): super().__init__() - self._status = 1 - self._memory = psutil.virtual_memory() + self._status = 0 def implement(self, adapter, simulation, simdata): super().implement(adapter, simulation, simdata) self._simdata = simdata def get_next_checkpoint(self): - return self._status + return self._status + self.step def run_checkpoint(self, kwargs=None): - # If threshold is reached Flush data - if self._memory.percent > self.threshold: - self._simdata.result.flush() - - self._status += 1 + # Flush data + self._simdata.result.flush() + if self._status: + mem_size_after, mem_peak = tracemalloc.get_traced_memory() + assert (mem_peak/(1024**2) - mem_size_after/(1024**2)) > 250 + self._status += self.step return self._status self.network.simulations.test.devices["new"] = dict( @@ -638,12 +641,11 @@ def run_checkpoint(self, kwargs=None): "strategy": "cell_model", "cell_models": ["A", "B", "C"], }, - threshold=75, + step=5000, ) self.network.run_simulation("test", "out.nio") - def tearDown(self): - # Memory after the test - after_mem = self.process.memory_info().rss - delta = (after_mem - self.before_mem) / (1024**2) - print(f"\n{self._testMethodName} used {delta:.2f} MB") + mem_size, mem_peak = tracemalloc.get_traced_memory() + self.assertLess(mem_peak/ (1024 ** 2),600,"Memory overflow") + + From bbe381470586b379d19f1692c4c8c008fdc9205c Mon Sep 17 00:00:00 2001 From: filimarc Date: Thu, 16 Apr 2026 16:44:46 +0200 Subject: [PATCH 09/18] fix: nest installation issue --- devtools/test_examples.sh | 9 --------- examples/project.json | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/devtools/test_examples.sh b/devtools/test_examples.sh index 10927e7f..566031b8 100644 --- a/devtools/test_examples.sh +++ b/devtools/test_examples.sh @@ -16,15 +16,6 @@ for ((i=0; i /dev/null - if [ $? -ne 0 ]; then - output_results+=("1") - cd .. || exit 1 - continue - fi extra="--env-file $NEST_FOLDER/install/bin/nest_vars.sh" fi uv sync diff --git a/examples/project.json b/examples/project.json index 38c4366c..7f6ca2ad 100644 --- a/examples/project.json +++ b/examples/project.json @@ -11,6 +11,7 @@ } }, "sync": { + "dependsOn": ["bsb-nest:install-nest"], "executor": "@nxlv/python:sync", "options": {} }, @@ -48,6 +49,7 @@ "cache": true }, "test": { + "dependsOn": ["examples:sync"], "executor": "@nxlv/python:run-commands", "outputs": [], "options": { From d2a04ae1f4133693313f2f394a1dde325a3915dc Mon Sep 17 00:00:00 2001 From: filimarc Date: Thu, 16 Apr 2026 17:31:19 +0200 Subject: [PATCH 10/18] fix: test with MPI --- packages/bsb-neuron/tests/test_neuron.py | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/bsb-neuron/tests/test_neuron.py b/packages/bsb-neuron/tests/test_neuron.py index 00de2591..039b3b97 100644 --- a/packages/bsb-neuron/tests/test_neuron.py +++ b/packages/bsb-neuron/tests/test_neuron.py @@ -545,7 +545,7 @@ def setUp(self): p.parallel.gid_clear() for ct in self.network.cell_types.values(): ct.spatial.morphologies = ["3branch"] - + self.network.chunk_size=[50,50,50] hh_soma = { "cable_types": { "soma": { @@ -587,16 +587,15 @@ def setUp(self): C=ArborizedModel(model=hh_soma), ), connection_models=dict( - A_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), - A_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), - B_to_C=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), - C_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), - C_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn")]), + A_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), + A_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), + B_to_C=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), + C_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), + C_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), ), devices=devices, ) self.network.compile() - print(f"{self.network.simulations.test.duration}") self.process = psutil.Process(os.getpid()) # Baseline before the test self.before_mem = self.process.memory_info().rss @@ -606,7 +605,11 @@ def test_RAM_usage(self): the maximum peak of memory do not exceed 600 MB and that after every flush the usage is at least 250 MB below the peak.""" import tracemalloc + import os tracemalloc.start() + + flush_threshold = 250 /MPI.get_size() + total_threshold = 600 /MPI.get_size() @config.node class SpikeController( VoltageRecorder, @@ -630,7 +633,7 @@ def run_checkpoint(self, kwargs=None): self._simdata.result.flush() if self._status: mem_size_after, mem_peak = tracemalloc.get_traced_memory() - assert (mem_peak/(1024**2) - mem_size_after/(1024**2)) > 250 + assert (mem_peak/(1024**2) - mem_size_after/(1024**2)) > flush_threshold self._status += self.step return self._status @@ -643,9 +646,11 @@ def run_checkpoint(self, kwargs=None): }, step=5000, ) - self.network.run_simulation("test", "out.nio") + rank = MPI.get_rank() + self.network.run_simulation("test", "out"+str(rank)+".nio") mem_size, mem_peak = tracemalloc.get_traced_memory() - self.assertLess(mem_peak/ (1024 ** 2),600,"Memory overflow") + self.assertLess(mem_peak/ (1024 ** 2),total_threshold,"Memory overflow") + os.remove("out"+str(rank)+".nio") From 3f219160d7e56ecfc51226bbb433ecc63b085f20 Mon Sep 17 00:00:00 2001 From: filimarc Date: Fri, 17 Apr 2026 11:47:11 +0200 Subject: [PATCH 11/18] chore: lint files --- packages/bsb-neuron/tests/test_neuron.py | 40 +++++++++++++----------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/bsb-neuron/tests/test_neuron.py b/packages/bsb-neuron/tests/test_neuron.py index 039b3b97..3dfdd5ad 100644 --- a/packages/bsb-neuron/tests/test_neuron.py +++ b/packages/bsb-neuron/tests/test_neuron.py @@ -545,7 +545,7 @@ def setUp(self): p.parallel.gid_clear() for ct in self.network.cell_types.values(): ct.spatial.morphologies = ["3branch"] - self.network.chunk_size=[50,50,50] + self.network.chunk_size = [50, 50, 50] hh_soma = { "cable_types": { "soma": { @@ -587,11 +587,11 @@ def setUp(self): C=ArborizedModel(model=hh_soma), ), connection_models=dict( - A_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), - A_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), - B_to_C=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), - C_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), - C_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn",delay=1.0)]), + A_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn", delay=1.0)]), + A_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn", delay=1.0)]), + B_to_C=TransceiverModel(synapses=[dict(synapse="ExpSyn", delay=1.0)]), + C_to_A=TransceiverModel(synapses=[dict(synapse="ExpSyn", delay=1.0)]), + C_to_B=TransceiverModel(synapses=[dict(synapse="ExpSyn", delay=1.0)]), ), devices=devices, ) @@ -602,16 +602,18 @@ def setUp(self): def test_RAM_usage(self): """This test run a simulation that records around 1.1 GB of data and check that - the maximum peak of memory do not exceed 600 MB and that after every flush - the usage is at least 250 MB below the peak.""" - import tracemalloc + the maximum peak of memory do not exceed 600 MB and that after every flush + the usage is at least 250 MB below the peak.""" import os + import tracemalloc + tracemalloc.start() - flush_threshold = 250 /MPI.get_size() - total_threshold = 600 /MPI.get_size() + flush_threshold = 250 / MPI.get_size() + total_threshold = 600 / MPI.get_size() + @config.node - class SpikeController( + class RAMController( VoltageRecorder, classmap_entry="ram_controller", ): @@ -633,13 +635,15 @@ def run_checkpoint(self, kwargs=None): self._simdata.result.flush() if self._status: mem_size_after, mem_peak = tracemalloc.get_traced_memory() - assert (mem_peak/(1024**2) - mem_size_after/(1024**2)) > flush_threshold + assert ( + mem_peak / (1024**2) - mem_size_after / (1024**2) + ) > flush_threshold self._status += self.step return self._status self.network.simulations.test.devices["new"] = dict( - device="spike_controller", + device="ram_controller", targetting={ "strategy": "cell_model", "cell_models": ["A", "B", "C"], @@ -647,10 +651,8 @@ def run_checkpoint(self, kwargs=None): step=5000, ) rank = MPI.get_rank() - self.network.run_simulation("test", "out"+str(rank)+".nio") + self.network.run_simulation("test", "out" + str(rank) + ".nio") mem_size, mem_peak = tracemalloc.get_traced_memory() - self.assertLess(mem_peak/ (1024 ** 2),total_threshold,"Memory overflow") - os.remove("out"+str(rank)+".nio") - - + self.assertLess(mem_peak / (1024**2), total_threshold, "Memory overflow") + os.remove("out" + str(rank) + ".nio") From b84664a2d7eab0014cc6cc34699f1a6ae34fe752 Mon Sep 17 00:00:00 2001 From: filimarc Date: Fri, 8 May 2026 15:40:02 +0200 Subject: [PATCH 12/18] fix: update to new nest 3.10 --- packages/bsb-nest/tests/test_nest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bsb-nest/tests/test_nest.py b/packages/bsb-nest/tests/test_nest.py index e7128ef9..30c0be36 100644 --- a/packages/bsb-nest/tests/test_nest.py +++ b/packages/bsb-nest/tests/test_nest.py @@ -8,7 +8,7 @@ from bsb.core import Scaffold from bsb.services import MPI from bsb_test import NumpyTestCase, RandomStorageFixture, get_test_config -from nest.lib.hl_api_exceptions import NESTErrors +from nest import NESTErrors from scipy.optimize import curve_fit from bsb_nest import NestAdapter From a830db74bdcb2db8b8a9f5d57760c6452adf295b Mon Sep 17 00:00:00 2001 From: filimarc Date: Mon, 11 May 2026 10:27:27 +0200 Subject: [PATCH 13/18] fix: add check on nest version --- packages/bsb-nest/tests/test_nest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/bsb-nest/tests/test_nest.py b/packages/bsb-nest/tests/test_nest.py index 30c0be36..d1b5d66f 100644 --- a/packages/bsb-nest/tests/test_nest.py +++ b/packages/bsb-nest/tests/test_nest.py @@ -8,7 +8,12 @@ from bsb.core import Scaffold from bsb.services import MPI from bsb_test import NumpyTestCase, RandomStorageFixture, get_test_config -from nest import NESTErrors +from packaging.version import Version + +if Version(nest.__version__) >= Version("3.10"): + from nest import NESTErrors +else: + from nest.lib.hl_api_exceptions import NESTErrors from scipy.optimize import curve_fit from bsb_nest import NestAdapter From 36898ebc743949af63f6c196741bdb46b14c9059 Mon Sep 17 00:00:00 2001 From: filimarc Date: Mon, 11 May 2026 10:56:38 +0200 Subject: [PATCH 14/18] fix: add a third sim to the test --- packages/bsb-core/tests/test_simulation.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/bsb-core/tests/test_simulation.py b/packages/bsb-core/tests/test_simulation.py index 66355613..e8844d62 100644 --- a/packages/bsb-core/tests/test_simulation.py +++ b/packages/bsb-core/tests/test_simulation.py @@ -565,7 +565,7 @@ def test_two_controllers(self): "Spike times in last segment fall outside the expected range (90-100).", ) - def test_checkpoints_with_double_sim(self): + def test_checkpoints_with_triple_sim(self): """This test checks that if two simulations are run the results are written in two separate blocks inside the same file""" self.network.simulations.test.devices["new_recorder"] = dict( @@ -628,16 +628,19 @@ def test_checkpoints_with_double_sim(self): ), ) self.network.simulations["sim_2"] = ArborSimulation(sim_2) + self.network.simulations["sim_3"] = ArborSimulation(sim_2) nio_file = "out.nio" self.network.run_simulation("test", output_filename=nio_file) self.network.run_simulation("sim_2", output_filename=nio_file) + self.network.run_simulation("sim_3", output_filename=nio_file) written_results = io.NixIO(nio_file, "ro") blocks = written_results.read_all_blocks() - self.assertEqual(len(blocks), 2) + self.assertEqual(len(blocks), 3) self.assertEqual(len(blocks[0].segments), 11) self.assertEqual(len(blocks[1].segments), 6) + self.assertEqual(len(blocks[2].segments), 6) import os os.remove(nio_file) From bbfa58c7353a98e7cee1b5a6bd99b7e488e8d9d9 Mon Sep 17 00:00:00 2001 From: filimarc Date: Wed, 13 May 2026 16:49:50 +0200 Subject: [PATCH 15/18] fix: add test for RAM usage in bsb-core | add alternatives for analogs and spiket methods --- packages/bsb-core/bsb/simulation/results.py | 5 ++ packages/bsb-core/tests/test_simulation.py | 66 ++++++++++++++++++++- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/packages/bsb-core/bsb/simulation/results.py b/packages/bsb-core/bsb/simulation/results.py index a36d6e5d..f1b8f8e1 100644 --- a/packages/bsb-core/bsb/simulation/results.py +++ b/packages/bsb-core/bsb/simulation/results.py @@ -35,11 +35,15 @@ def __init__(self, simulation, filename=None): def analogsignals(self): if hasattr(self, "block"): return self.block.segments[0].analogsignals + else: + return [] @property def spiketrains(self): if hasattr(self, "block"): return self.block.segments[0].spiketrains + else: + return [] def add(self, recorder): self.recorders.append(recorder) @@ -52,6 +56,7 @@ def create_recorder(self, flush: typing.Callable[["neo.core.Segment"], None]): def flush(self): from neo import Segment, io + segment = Segment() for recorder in self.recorders: try: diff --git a/packages/bsb-core/tests/test_simulation.py b/packages/bsb-core/tests/test_simulation.py index e8844d62..cad51764 100644 --- a/packages/bsb-core/tests/test_simulation.py +++ b/packages/bsb-core/tests/test_simulation.py @@ -3,12 +3,13 @@ import numpy as np from bsb_arbor import ArborSimulation, SpikeRecorder from bsb_test import FixedPosConfigFixture, NumpyTestCase, RandomStorageFixture -from neo import io +from neo import AnalogSignal, io from bsb import ( MPI, AttributeMissingError, Scaffold, + SimulationResult, config, get_simulation_adapter, options, @@ -313,7 +314,6 @@ def run_checkpoint(self, kwargs=None): return self._status -@unittest.skipIf(MPI.get_size() > 1, "Skipped during parallel testing.") class TestAdapterControllers( FixedPosConfigFixture, RandomStorageFixture, @@ -428,6 +428,7 @@ def test_controller_registration(self): "Only rec_15 device should be registered", ) + @unittest.skipIf(MPI.get_size() > 1, "Skipped during parallel testing.") def test_registration_of_listener(self): self.network.simulations.test.devices["rec_15"] = dict( device="spike_controller", @@ -480,6 +481,7 @@ def get_next_checkpoint(self): with self.assertRaises(AttributeMissingError): adapter.prepare(sim) + @unittest.skipIf(MPI.get_size() > 1, "Skipped during parallel testing.") def test_record_checkpoint(self): """Create a test with an AdapterController that flushes every 10 steps, so with a simulation of 100 of duration it will create 10 segments plus @@ -517,6 +519,7 @@ def test_record_checkpoint(self): "Spike times in segment 6 fall outside the expected range (60–70).", ) + @unittest.skipIf(MPI.get_size() > 1, "Skipped during parallel testing.") def test_two_controllers(self): """Test two AdapterController instances together, one configured with 15 steps and the other with 40 steps, it will flush at [ 15,30,40,45,60,75,80,90]. @@ -565,6 +568,7 @@ def test_two_controllers(self): "Spike times in last segment fall outside the expected range (90-100).", ) + @unittest.skipIf(MPI.get_size() > 1, "Skipped during parallel testing.") def test_checkpoints_with_triple_sim(self): """This test checks that if two simulations are run the results are written in two separate blocks inside the same file""" @@ -644,3 +648,61 @@ def test_checkpoints_with_triple_sim(self): import os os.remove(nio_file) + + def test_RAM_usage(self): + """ + This test writes over 7 GB of data in 60 segments of 120 MB each. + + It verifies that memory usage, after the initial steps, remains stable + within a 5 MB threshold. It also checks that peak memory usage never + exceeds 500 MB. + If PLOT_GRAPH is set to True, it will also plot the graph of memory + after every flush. + """ + import os + import tracemalloc + + import matplotlib.pyplot as plt + from quantities import ms + + tracemalloc.start() + PLOT_GRAPH = False + MB = 1024 * 1024 + total_threshold = 500 + rank = MPI.get_rank() + + sim = self.network.simulations.test + nio_file = "out" + str(rank) + ".nio" + my_result = SimulationResult(sim, nio_file) + + def my_flush(segment): + # Creates a signal of 120 MB + signal = np.ones(15 * MB) + segment.analogsignals.append( + AnalogSignal(signal, sampling_period=0.1 * ms, units="mV") + ) + + my_result.create_recorder(my_flush) + + num_samples = 60 + mem_size = np.zeros(num_samples) + for sample in range(num_samples): + my_result.flush() + mem_size_after, mem_peak = tracemalloc.get_traced_memory() + mem_size[sample] = mem_size_after / MB + + _, mem_peak = tracemalloc.get_traced_memory() + mean = np.mean(mem_size[5::]) + max_dev = np.max(np.abs(mem_size[5::] - mean)) + if PLOT_GRAPH: + plt.plot(mem_size) + + plt.xlabel("step") + plt.ylabel("Mem Size") + plt.ylim(0, 400) + + plt.grid(True) + plt.show() + self.assertClose(0, max_dev, atol=5.0) + self.assertLess(mem_peak / MB, total_threshold) + os.remove("out" + str(rank) + ".nio") From f28891c1622076894650c983706c211057e59bd1 Mon Sep 17 00:00:00 2001 From: filimarc Date: Wed, 13 May 2026 16:57:32 +0200 Subject: [PATCH 16/18] fix: remove test in bsb-neuron --- packages/bsb-core/tests/test_simulation.py | 2 +- packages/bsb-neuron/tests/test_neuron.py | 55 ---------------------- 2 files changed, 1 insertion(+), 56 deletions(-) diff --git a/packages/bsb-core/tests/test_simulation.py b/packages/bsb-core/tests/test_simulation.py index cad51764..8d7452e2 100644 --- a/packages/bsb-core/tests/test_simulation.py +++ b/packages/bsb-core/tests/test_simulation.py @@ -657,7 +657,7 @@ def test_RAM_usage(self): within a 5 MB threshold. It also checks that peak memory usage never exceeds 500 MB. If PLOT_GRAPH is set to True, it will also plot the graph of memory - after every flush. + values collected after every flush. """ import os import tracemalloc diff --git a/packages/bsb-neuron/tests/test_neuron.py b/packages/bsb-neuron/tests/test_neuron.py index 3dfdd5ad..da72f2e1 100644 --- a/packages/bsb-neuron/tests/test_neuron.py +++ b/packages/bsb-neuron/tests/test_neuron.py @@ -600,59 +600,4 @@ def setUp(self): # Baseline before the test self.before_mem = self.process.memory_info().rss - def test_RAM_usage(self): - """This test run a simulation that records around 1.1 GB of data and check that - the maximum peak of memory do not exceed 600 MB and that after every flush - the usage is at least 250 MB below the peak.""" - import os - import tracemalloc - - tracemalloc.start() - - flush_threshold = 250 / MPI.get_size() - total_threshold = 600 / MPI.get_size() - - @config.node - class RAMController( - VoltageRecorder, - classmap_entry="ram_controller", - ): - step = config.attr(type=float, required=True) - - def __init__(self, **kwargs): - super().__init__() - self._status = 0 - - def implement(self, adapter, simulation, simdata): - super().implement(adapter, simulation, simdata) - self._simdata = simdata - - def get_next_checkpoint(self): - return self._status + self.step - - def run_checkpoint(self, kwargs=None): - # Flush data - self._simdata.result.flush() - if self._status: - mem_size_after, mem_peak = tracemalloc.get_traced_memory() - assert ( - mem_peak / (1024**2) - mem_size_after / (1024**2) - ) > flush_threshold - - self._status += self.step - return self._status - - self.network.simulations.test.devices["new"] = dict( - device="ram_controller", - targetting={ - "strategy": "cell_model", - "cell_models": ["A", "B", "C"], - }, - step=5000, - ) - rank = MPI.get_rank() - self.network.run_simulation("test", "out" + str(rank) + ".nio") - mem_size, mem_peak = tracemalloc.get_traced_memory() - self.assertLess(mem_peak / (1024**2), total_threshold, "Memory overflow") - os.remove("out" + str(rank) + ".nio") From 2ced6356eb334b6d4d1e02cf2b92b5333e68d6bc Mon Sep 17 00:00:00 2001 From: filimarc Date: Wed, 13 May 2026 17:05:20 +0200 Subject: [PATCH 17/18] fix: lint files --- packages/bsb-neuron/tests/test_neuron.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/bsb-neuron/tests/test_neuron.py b/packages/bsb-neuron/tests/test_neuron.py index da72f2e1..63a905a3 100644 --- a/packages/bsb-neuron/tests/test_neuron.py +++ b/packages/bsb-neuron/tests/test_neuron.py @@ -2,7 +2,7 @@ import unittest from copy import copy -from bsb import MPI, Scaffold, config, get_simulation_adapter +from bsb import MPI, Scaffold, get_simulation_adapter from bsb_test import ( ConfigFixture, MorphologiesFixture, @@ -13,7 +13,6 @@ from bsb_neuron.cell import ArborizedModel from bsb_neuron.connection import TransceiverModel -from bsb_neuron.devices import VoltageRecorder class TestNeuronMinimal( From d100767cfa194c45ecf5c7176acbaaf28d936e5b Mon Sep 17 00:00:00 2001 From: filimarc Date: Thu, 14 May 2026 12:05:20 +0200 Subject: [PATCH 18/18] fix: rollback nest-installation file --- devtools/install-nest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devtools/install-nest.sh b/devtools/install-nest.sh index 9068c35d..4207f02b 100755 --- a/devtools/install-nest.sh +++ b/devtools/install-nest.sh @@ -1,6 +1,6 @@ # Get Nest installation folder -SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )" if [ -z "$NEST_FOLDER" ]; then + SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"; NEST_FOLDER="$(dirname $SCRIPT_DIR)/.nx/installation/nest"; fi # Get NEST version @@ -13,7 +13,7 @@ if [ -f "$INSTALLATION_FOLDER/bin/nest_vars.sh" ]; then fi # Lock check and installation to prevent concurrent file edition -LOCK_FILE="$SCRIPT_DIR/bsb-nest.lock" +LOCK_FILE="/tmp/bsb-nest.lock" # Remove lock file on exit trap 'rm -f "$LOCK_FILE"' EXIT # Wait to be able to create file