From 147020f50576549b8feba410028b7a9c21ae6f38 Mon Sep 17 00:00:00 2001 From: Graham Kimbell <88666181+g-kimbell@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:35:14 +0100 Subject: [PATCH 1/2] Allow multiple runs and one combined summary --- pyflowbatt/analysis.py | 86 +++++++++++++++++--- pyflowbatt/cli.py | 118 ++++++++++++++++++++++++++-- tests/test_cli.py | 173 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 359 insertions(+), 18 deletions(-) diff --git a/pyflowbatt/analysis.py b/pyflowbatt/analysis.py index 2ffd7be..50a5839 100644 --- a/pyflowbatt/analysis.py +++ b/pyflowbatt/analysis.py @@ -6,6 +6,7 @@ import re import uuid import zipfile +from collections.abc import Sequence from pathlib import Path, PurePosixPath from typing import Literal from urllib.parse import unquote @@ -1293,7 +1294,7 @@ def _merged_row_order(indexes: list[pd.Index]) -> list[str]: def merge_summaries( - summary_xlsxs: list[str | Path], + summary_xlsxs: Sequence[str | Path], sample_ids_by_sample_folder: dict[Path, str] | None = None, ) -> pd.DataFrame: """Merge all summary sheets into one mega summary.""" @@ -1326,6 +1327,73 @@ def merge_summaries( return df[cols] +def merge_folder_summaries( + summaries_by_folder: dict[str, list[Path]], + sample_ids_by_sample_folder: dict[Path, str] | None = None, +) -> pd.DataFrame: + """Each folder's Average and Std over its samples, side by side.""" + merged = { + label: merge_summaries(paths, sample_ids_by_sample_folder).set_index("Quantity") + for label, paths in summaries_by_folder.items() + } + df = pd.DataFrame( + {f"{label} {stat}": m[stat] for label, m in merged.items() for stat in ("Average", "Std")} + ) + order = _merged_row_order([m.index for m in merged.values()]) + return df.reindex(order).rename_axis("Quantity").reset_index() + + +def write_summary_xlsx(path: Path, sheets: dict[str, pd.DataFrame]) -> None: + """Write each table to its own autofitted sheet.""" + with pd.ExcelWriter(path, engine="xlsxwriter") as writer: + for name, df in sheets.items(): + df.to_excel(writer, index=False, sheet_name=name) + writer.sheets[name].autofit() + + +def folder_labels(folders: list[Path]) -> dict[Path, str]: + """Label each folder by its name, with as many parent folders as needed to tell them apart.""" + depth = dict.fromkeys(folders, 1) + while True: + labels = {f: Path(*f.parts[-depth[f] :]).as_posix() for f in folders} + clashes = [ + f + for f in folders + if list(labels.values()).count(labels[f]) > 1 and depth[f] < len(f.parts) + ] + if not clashes: + return labels + for f in clashes: + depth[f] += 1 + + +def write_multi_folder_summary( + sample_ids_by_folder: dict[Path, dict[Path, str]], path: Path +) -> bool: + """Write all samples' summaries, and each folder's averages, from several folders into path. + + Returns False if no folder has any sample summaries. + """ + sample_ids = {sf: sid for ids in sample_ids_by_folder.values() for sf, sid in ids.items()} + labels = folder_labels(list(sample_ids_by_folder)) + summaries_by_folder = { + labels[folder]: summaries + for folder, ids in sample_ids_by_folder.items() + if (summaries := [s for sf in ids if (s := sf / "results" / "summary.xlsx").exists()]) + } + if not summaries_by_folder: + return False + all_summaries = [s for summaries in summaries_by_folder.values() for s in summaries] + write_summary_xlsx( + path, + { + "Summary": merge_summaries(all_summaries, sample_ids), + "By folder": merge_folder_summaries(summaries_by_folder, sample_ids), + }, + ) + return True + + def analyse_all_samples( # noqa: PLR0913 folder: str | Path, *, @@ -1334,8 +1402,8 @@ def analyse_all_samples( # noqa: PLR0913 max_folder_searches: int = DEFAULT_SEARCH, zip_output: bool = False, reporter: Reporter | None = None, -) -> None: - """Take a folder and run all analysis. +) -> dict[Path, str]: + """Take a folder and run all analysis, returns the sample ID of each sample folder. If ``zip_output`` is True, everything is zipped into a single zip file next to ``folder`` once analysis finishes, for uploading to Zenodo. The zip holds an RO-Crate @@ -1356,7 +1424,7 @@ def analyse_all_samples( # noqa: PLR0913 ) if len(sample_folders) == 0: logger.error("No sample folders found in %s", folder) - return + return {} configs_by_sample_folder: dict[Path, PyFlowBattConfig] = { sample_folder: PyFlowBattConfig.load(sample_folder) for sample_folder in sample_folders } @@ -1401,12 +1469,7 @@ def analyse_all_samples( # noqa: PLR0913 folder, max_search_depth, max_folder_searches, search_config ) df = merge_summaries(summaries, sample_ids_by_sample_folder) - writer = pd.ExcelWriter(folder / "combined_summary.xlsx", engine="xlsxwriter") - df.to_excel(writer, index=False, sheet_name="Summary") - workbook = writer.book - worksheet = writer.sheets["Summary"] - worksheet.autofit() - workbook.close() + write_summary_xlsx(folder / "combined_summary.xlsx", {"Summary": df}) logger.debug("Combined all the results into one big summary") if zip_output: @@ -1454,6 +1517,7 @@ def analyse_all_samples( # noqa: PLR0913 crate_path.unlink(missing_ok=True) reporter.finish() + return sample_ids_by_sample_folder def dry_analyse_all_samples( @@ -1504,7 +1568,7 @@ def dry_analyse_all_samples( logger.info("Results would be saved as battery data format in '%s' files.", save_format) if len(sample_folders) > 1: logger.info( - "Then I would combine all the summaries into one 'combined_results.xlsx' " + "Then I would combine all the summaries into one 'combined_summary.xlsx' " "at the root '%s'.", folder.name, ) diff --git a/pyflowbatt/cli.py b/pyflowbatt/cli.py index af33929..d658c8e 100644 --- a/pyflowbatt/cli.py +++ b/pyflowbatt/cli.py @@ -70,8 +70,8 @@ def analyse( # noqa: PLR0913 max_folder_searches: int = 10000, max_search_depth: int = 6, verbose: bool = False, -) -> None: - """Run PyFlowBatt on a folder.""" +) -> dict[Path, str]: + """Run PyFlowBatt on a folder, returns the sample ID of each analysed sample folder.""" import matplotlib as mpl # noqa: PLC0415 # Plots are only saved, and a GUI backend breaks when the live display's thread collects them @@ -84,7 +84,7 @@ def analyse( # noqa: PLR0913 if not folderpath.is_dir(): with _console_logging(reporter): logger.error("%s is not a directory", folderpath) - return + return {} log_path = None if dry else folderpath / LOG_FILENAME with _console_logging(reporter, verbose=verbose, log_path=log_path): try: @@ -97,7 +97,7 @@ def analyse( # noqa: PLR0913 zip_output=zip_output, ) else: - analyse_all_samples( + return analyse_all_samples( folderpath, max_folder_searches=max_folder_searches, max_search_depth=max_search_depth, @@ -107,6 +107,92 @@ def analyse( # noqa: PLR0913 except ConfigError as e: # The message names the file and line, a traceback would only hide it logger.error("%s", e) # noqa: TRY400 + return {} + + +def _combined_summary_path(path: str, *, dry: bool) -> Path: + """Resolve --combined-summary to an .xlsx file, raising ValueError if it can't be written.""" + from pyflowbatt.analysis import check_path_writable # noqa: PLC0415 + + summary_path = Path(path).resolve() + if summary_path.is_dir(): + summary_path /= "combined_summary.xlsx" + elif summary_path.suffix.lower() != ".xlsx": + msg = f"--combined-summary must be a folder or an .xlsx file, not {path}" + raise ValueError(msg) + if not summary_path.parent.is_dir(): + msg = f"Cannot write {summary_path.name}: {summary_path.parent} is not a directory" + raise ValueError(msg) + if not dry: + try: + check_path_writable(summary_path) + except PermissionError as e: + raise ValueError(str(e)) from e + return summary_path + + +def analyse_folders( # noqa: PLR0913 + folders: list[str], + *, + combined_summary: str | None = None, + dry: bool = False, + zip_output: bool = False, + max_folder_searches: int = 10000, + max_search_depth: int = 6, + verbose: bool = False, +) -> None: + """Run PyFlowBatt on each folder, then combine all their summaries if asked to.""" + from pyflowbatt.analysis import write_multi_folder_summary # noqa: PLC0415 + + paths = list(dict.fromkeys(Path(f).resolve() for f in folders)) + with _console_logging(RichReporter()): + if bad := [p for p in paths if not p.is_dir()]: + logger.error("Not a directory: %s", ", ".join(map(str, bad))) + return + if nested := [(a, b) for a in paths for b in paths if a != b and b.is_relative_to(a)]: + # Samples in both would be analysed twice and counted twice in the averages + logger.error("%s is inside %s, pass only one of them", *nested[0][::-1]) + return + try: + summary_path = ( + _combined_summary_path(combined_summary, dry=dry) if combined_summary else None + ) + except ValueError as e: + logger.error("%s", e) # noqa: TRY400 + return + + sample_ids_by_folder = { + path: analyse( + str(path), + dry=dry, + zip_output=zip_output, + max_folder_searches=max_folder_searches, + max_search_depth=max_search_depth, + verbose=verbose, + ) + for path in paths + } + if summary_path is None: + return + + reporter = RichReporter(verbose=verbose) + with _console_logging(reporter, verbose=verbose): + if dry: + logger.info( + "Then I would combine the summaries from all %d folders into '%s'.", + len(paths), + summary_path, + ) + return + with ( + reporter.section(f"Across {len(paths)} folders"), + reporter.task("Combined summary", detail=summary_path.name), + ): + written = write_multi_folder_summary(sample_ids_by_folder, summary_path) + if not written: + logger.error("No sample summaries to combine") + if written: + logger.info("\nCombined summary of all folders: %s", summary_path) def main() -> None: @@ -131,7 +217,22 @@ def main() -> None: "--dry", action="store_true", help="Don't do any analysis, just check the folders" ) parser.add_argument( - "--folder", type=str, help="Folder to analyse, if not specified, analyse current folder" + "--folder", + type=str, + nargs="+", + help=( + "Folder(s) to analyse, if not specified, analyse current folder. " + "Several folders are analysed one after another." + ), + ) + parser.add_argument( + "--combined-summary", + type=str, + metavar="PATH", + help=( + "Also combine the summaries of every folder analysed into one Excel file at PATH, " + "an .xlsx file or a folder to write combined_summary.xlsx into" + ), ) parser.add_argument( "--zip", @@ -169,7 +270,12 @@ def main() -> None: kwargs["max_folder_searches"] = args.max_folder_searches if args.max_search_depth is not None: kwargs["max_search_depth"] = args.max_search_depth - analyse(folder=args.folder, dry=args.dry, zip_output=args.zip, verbose=args.verbose, **kwargs) + kwargs.update(dry=args.dry, zip_output=args.zip, verbose=args.verbose) + if args.combined_summary or (args.folder and len(args.folder) > 1): + folders = args.folder or [str(Path.cwd())] + analyse_folders(folders, combined_summary=args.combined_summary, **kwargs) + else: + analyse(folder=args.folder[0] if args.folder else None, **kwargs) if __name__ == "__main__": diff --git a/tests/test_cli.py b/tests/test_cli.py index 42cf235..4a1a1c6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,12 +1,13 @@ """Tests for the PyFlowBatt CLI, in particular `pyflowbatt init`.""" import zipfile +from collections.abc import Callable from pathlib import Path import pytest from pyflowbatt import cli -from pyflowbatt.analysis import zip_folder +from pyflowbatt.analysis import folder_labels, zip_folder def test_init_config_writes_template(tmp_path: Path) -> None: @@ -147,3 +148,173 @@ def test_log_file_has_debug_only_when_verbose(tmp_path: Path, *, verbose: bool) cli.analyse(folder=str(tmp_path), verbose=verbose) log = (tmp_path / "pyflowbatt.log").read_text(encoding="utf-8") assert ("Loading PyFlowBatt config from" in log) is verbose + + +@pytest.mark.parametrize( + ("argv", "folders", "combined_summary"), + [ + (["--folder", "x", "y"], ["x", "y"], None), + (["--folder", "x", "--combined-summary", "out.xlsx"], ["x"], "out.xlsx"), + ], +) +def test_main_dispatches_analyse_folders( + monkeypatch: pytest.MonkeyPatch, + argv: list[str], + folders: list[str], + combined_summary: str | None, +) -> None: + """Several folders, or --combined-summary, dispatch to analyse_folders.""" + called_with: dict = {} + + def fake_analyse_folders(folders: list[str], **kwargs: object) -> None: + called_with.update(kwargs, folders=folders) + + monkeypatch.setattr(cli, "analyse_folders", fake_analyse_folders) + monkeypatch.setattr("sys.argv", ["pyflowbatt", *argv]) + + cli.main() + + assert called_with == { + "folders": folders, + "combined_summary": combined_summary, + "dry": False, + "zip_output": False, + "verbose": False, + } + + +def _write_summary(sample_folder: Path, values: dict[str, float]) -> None: + import pandas as pd # noqa: PLC0415 + + path = sample_folder / "results" / "summary.xlsx" + path.parent.mkdir(parents=True) + df = pd.DataFrame({"Value": list(values.values()), "Error": None}, index=list(values)) + df.to_excel(path, sheet_name="Summary") + + +def _fake_analyse( + summaries: dict[str, dict[str, dict[str, float]]], +) -> Callable[..., dict[Path, str]]: + """Analyse that writes the given summaries per sample instead of analysing data.""" + + def fake(folder: str, **_kwargs: object) -> dict[Path, str]: + ids = {} + for sample, values in summaries[Path(folder).name].items(): + _write_summary(Path(folder) / sample, values) + ids[Path(folder) / sample] = sample + return ids + + return fake + + +def _make_folders(tmp_path: Path, names: list[str]) -> list[str]: + for name in names: + (tmp_path / name).mkdir(parents=True) + return [str(tmp_path / name) for name in names] + + +def test_analyse_folders_combines_summaries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Every sample, and each folder's average, go into the --combined-summary file.""" + import pandas as pd # noqa: PLC0415 + + summaries = { + "x": {"x1": {"OCV": 1.0, "CE": 90.0}, "x2": {"OCV": 3.0, "CE": 92.0}}, + "y": {"y1": {"OCV": 5.0, "Extra": 7.0}}, + } + folders = _make_folders(tmp_path / "data", list(summaries)) + (tmp_path / "elsewhere").mkdir() + monkeypatch.setattr(cli, "analyse", _fake_analyse(summaries)) + + cli.analyse_folders(folders, combined_summary=str(tmp_path / "elsewhere" / "all.xlsx")) + + sheets = pd.read_excel(tmp_path / "elsewhere" / "all.xlsx", sheet_name=None, index_col=0) + assert list(sheets) == ["Summary", "By folder"] + assert list(sheets["Summary"].columns) == ["Average", "Std", "x1", "x2", "y1"] + assert sheets["Summary"].loc["OCV", "Average"] == 3.0 + by_folder = sheets["By folder"] + assert list(by_folder.columns) == ["x Average", "x Std", "y Average", "y Std"] + assert list(by_folder.index) == ["OCV", "Extra", "CE"] + assert by_folder.loc["OCV", "x Average"] == 2.0 + assert by_folder.loc["OCV", "y Average"] == 5.0 + assert pd.isna(by_folder.loc["CE", "y Average"]) + + +def test_analyse_folders_combined_summary_into_folder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A folder given as --combined-summary gets combined_summary.xlsx inside it.""" + folders = _make_folders(tmp_path, ["x", "y"]) + monkeypatch.setattr(cli, "analyse", _fake_analyse({"x": {"x1": {"OCV": 1.0}}, "y": {}})) + + cli.analyse_folders(folders, combined_summary=str(tmp_path)) + + assert (tmp_path / "combined_summary.xlsx").exists() + + +def test_analyse_folders_without_combined_summary_writes_none( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Several folders are each analysed, with no combined file unless asked for.""" + folders = _make_folders(tmp_path, ["x", "y"]) + analysed = [] + monkeypatch.setattr(cli, "analyse", lambda folder, **_kwargs: analysed.append(folder) or {}) + + cli.analyse_folders(folders) + + assert analysed == [str(Path(f).resolve()) for f in folders] + assert list(tmp_path.glob("*.xlsx")) == [] + + +def test_analyse_folders_dry_writes_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A dry run writes no combined summary.""" + folders = _make_folders(tmp_path, ["x", "y"]) + monkeypatch.setattr(cli, "analyse", lambda *_args, **_kwargs: {}) + + cli.analyse_folders(folders, combined_summary=str(tmp_path), dry=True) + + assert not (tmp_path / "combined_summary.xlsx").exists() + + +def _fail(*_args: object, **_kwargs: object) -> None: + msg = "analyse should not run" + raise AssertionError(msg) + + +@pytest.mark.parametrize("target", ["missing/all.xlsx", "all.csv"]) +def test_analyse_folders_bad_combined_summary_fails_before_analysis( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + target: str, +) -> None: + """An unusable --combined-summary path is reported before any slow analysis runs.""" + folders = _make_folders(tmp_path, ["x", "y"]) + monkeypatch.setattr(cli, "analyse", _fail) + + with caplog.at_level("ERROR", logger="pyflowbatt"): + cli.analyse_folders(folders, combined_summary=str(tmp_path / target)) + + assert caplog.records + + +def test_analyse_folders_rejects_nested_folders( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A folder inside another would have its samples analysed and averaged twice.""" + (tmp_path / "x" / "inner").mkdir(parents=True) + monkeypatch.setattr(cli, "analyse", _fail) + + with caplog.at_level("ERROR", logger="pyflowbatt"): + cli.analyse_folders([str(tmp_path / "x"), str(tmp_path / "x" / "inner")]) + + assert "is inside" in caplog.text + + +def test_folder_labels_add_parents_only_where_names_clash() -> None: + """Folders are labelled by name, with parent folders added to tell same names apart.""" + folders = [Path("/a/run1"), Path("/b/run1"), Path("/c/x/run2")] + assert list(folder_labels(folders).values()) == ["a/run1", "b/run1", "run2"] From baf3c6f292fa4d25bfbc04c1fee9fb17d84ac12f Mon Sep 17 00:00:00 2001 From: Graham Kimbell <88666181+g-kimbell@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:05:40 +0100 Subject: [PATCH 2/2] Update readme --- README.md | 68 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0cf9a1a..ec86ddd 100644 --- a/README.md +++ b/README.md @@ -5,36 +5,82 @@ This Python package contains functions and a CLI for analysing data from the flo ## Installation -Use Python >3.10, and install with pip: +Use Python >=3.10, and install with pip: ``` -pip install git+https://github.com/empaeconversion/pyflowbatt.git +pip install pyflowbatt ``` ## Usage Use in a terminal (e.g. Command Prompt or PowerShell). -To analyse data and get a full summary and ro-crate, use `pyflowbatt --folder="path/to/your/folder"`. - -To zip everything at the end (e.g. to upload to Zenodo), use `pyflowbatt --folder="path/to/your/folder" --zip`. - -You can also navigate to a folder e.g. +To analyse the data in a folder, use +``` +pyflowbatt --folder "path/to/your/folder" +``` +Or go to a folder and just run `pyflowbatt`: ``` cd path/to/your/folder pyflowbatt ``` -PyFlowBatt will automatically scan for files like GCPL, EIS, LSV, CV, as well as BattInfo Converter .xlsx files, process the data, convert to battery data format, produce summary graphs and excel reports, create a BattInfo JSON-LD metadata file, and a top level ro-crate metadata file. -A full analysis can take a while. You can do a 'dry run' to check what files *would* be analysed: +PyFlowBatt processes the data, converts it to battery data format, makes summary graphs and Excel reports, and creates a BattINFO JSON-LD metadata file. + +### How to lay out data + +Make one folder per sample (a "sample folder") and put raw data in it (not in subfolders). PyFlowBatt recognises each measurement from its file name, so the names need to contain `_OCV_`, `_PEIS_`, `_LSV_`, `_GCPL_`, `_CVpre` or `_CVpost`. If your files are named differently, you can change this in a `pyflowbatt.toml` file (see below). + +You can run PyFlowBatt on one sample folder, or on a folder with many sample folders inside it: + ``` -pyflowbatt --dry +my-experiment/ <- run pyflowbatt here +├── 261028_abcd_cell1/ <- a sample folder +│ ├── ..._01_OCV_CE4.mpr +│ ├── ..._02_PEIS_CE4.mpr +│ ├── ..._04_GCPL_CE4.mpr +│ ├── battinfo.xlsx <- optional, from BattINFO Converter +│ └── results/ <- made by PyFlowBatt +├── 261028_abcd_cell2/ <- another sample folder +│ └── ... +└── combined_summary.xlsx <- made by PyFlowBatt +``` + +Name each sample folder starting with the date and your initials, e.g. `250115_abcd_...`. PyFlowBatt uses the folder name as the sample name in the summaries. + +Sample folders can be grouped into subfolders, but don't put a sample folder inside another sample folder. + +### Zipping for Zenodo + +If you use `--zip`, PyFlowBatt also makes a zip of everything, with a RO-Crate metadata file describing the files inside: +``` +pyflowbatt --folder "path/to/your/folder" --zip ``` +### Several folders at once + +You can analyse several folders in one go. Each folder is analysed as if run on its own (including the zip, if used). +``` +pyflowbatt --folder "path/to/X" "path/to/Y" "path/to/Z" ``` -pyflowbatt --dry --folder="path/to/your/folder" --zip +You can also combine all those summaries from multiple folders into one Excel file: add `--combined-summary` with an `.xlsx` file, or a folder to write `combined_summary.xlsx` into (`.` for the current folder). +``` +pyflowbatt --folder "path/to/X" "path/to/Y" "path/to/Z" --combined-summary "path/to/all_samples.xlsx" +``` +Its "Summary" sheet lists every sample from all the folders, and its "By folder" sheet puts each folder's average and standard deviation side by side. + +### Checking before a long run + +A full analysis can take a while. You can do a 'dry run' to check what files *would* be analysed, e.g.: +``` +pyflowbatt --dry +``` +``` +pyflowbatt --dry --folder "path/to/your/folder" --zip ``` This is useful for checking everything is in order before running a long analysis. +### Settings + You can override some settings by adding a `pyflowbatt.toml` file to a sample folder, parent folder, or your home folder. Use `pyflowbatt init` to create a template file and see what can be modified. ## Using in Python