Run on multiple folders

README.md

@@ -5,36 +5,82 @@ This Python package contains functions and a CLI for analysing data from the flo
 
 ## Installation
 
-Use Python >3.10, and install with pip:
+Use Python >=3.10, and install with pip:
 ```
-pip install git+https://github.com/empaeconversion/pyflowbatt.git
+pip install pyflowbatt
 ```
 
 ## Usage
 
 Use in a terminal (e.g. Command Prompt or PowerShell).
 
-To analyse data and get a full summary and ro-crate, use `pyflowbatt --folder="path/to/your/folder"`.
-
-To zip everything at the end (e.g. to upload to Zenodo), use `pyflowbatt --folder="path/to/your/folder" --zip`.
-
-You can also navigate to a folder e.g.
+To analyse the data in a folder, use
+```
+pyflowbatt --folder "path/to/your/folder"
+```
+Or go to a folder and just run `pyflowbatt`:
 ```
 cd path/to/your/folder
 pyflowbatt
 ```
-PyFlowBatt will automatically scan for files like GCPL, EIS, LSV, CV, as well as BattInfo Converter .xlsx files, process the data, convert to battery data format, produce summary graphs and excel reports, create a BattInfo JSON-LD metadata file, and a top level ro-crate metadata file.
 
-A full analysis can take a while. You can do a 'dry run' to check what files *would* be analysed:
+PyFlowBatt processes the data, converts it to battery data format, makes summary graphs and Excel reports, and creates a BattINFO JSON-LD metadata file.
+
+### How to lay out data
+
+Make one folder per sample (a "sample folder") and put raw data in it (not in subfolders). PyFlowBatt recognises each measurement from its file name, so the names need to contain `_OCV_`, `_PEIS_`, `_LSV_`, `_GCPL_`, `_CVpre` or `_CVpost`. If your files are named differently, you can change this in a `pyflowbatt.toml` file (see below).
+
+You can run PyFlowBatt on one sample folder, or on a folder with many sample folders inside it:
+
 ```
-pyflowbatt --dry
+my-experiment/               <- run pyflowbatt here
+├── 261028_abcd_cell1/        <- a sample folder
+│   ├── ..._01_OCV_CE4.mpr
+│   ├── ..._02_PEIS_CE4.mpr
+│   ├── ..._04_GCPL_CE4.mpr
+│   ├── battinfo.xlsx        <- optional, from BattINFO Converter
+│   └── results/             <- made by PyFlowBatt
+├── 261028_abcd_cell2/       <- another sample folder
+│   └── ...
+└── combined_summary.xlsx    <- made by PyFlowBatt
+```
+
+Name each sample folder starting with the date and your initials, e.g. `250115_abcd_...`. PyFlowBatt uses the folder name as the sample name in the summaries.
+
+Sample folders can be grouped into subfolders, but don't put a sample folder inside another sample folder.
+
+### Zipping for Zenodo
+
+If you use `--zip`, PyFlowBatt also makes a zip of everything, with a RO-Crate metadata file describing the files inside:
+```
+pyflowbatt --folder "path/to/your/folder" --zip
 ```
 
+### Several folders at once
+
+You can analyse several folders in one go. Each folder is analysed as if run on its own (including the zip, if used).
+```
+pyflowbatt --folder "path/to/X" "path/to/Y" "path/to/Z"
 ```
-pyflowbatt --dry --folder="path/to/your/folder" --zip
+You can also combine all those summaries from multiple folders into one Excel file: add `--combined-summary` with an `.xlsx` file, or a folder to write `combined_summary.xlsx` into (`.` for the current folder).
+```
+pyflowbatt --folder "path/to/X" "path/to/Y" "path/to/Z" --combined-summary "path/to/all_samples.xlsx"
+```
+Its "Summary" sheet lists every sample from all the folders, and its "By folder" sheet puts each folder's average and standard deviation side by side.
+
+### Checking before a long run
+
+A full analysis can take a while. You can do a 'dry run' to check what files *would* be analysed, e.g.:
+```
+pyflowbatt --dry
+```
+```
+pyflowbatt --dry --folder "path/to/your/folder" --zip
 ```
 This is useful for checking everything is in order before running a long analysis.
 
+### Settings
+
 You can override some settings by adding a `pyflowbatt.toml` file to a sample folder, parent folder, or your home folder. Use `pyflowbatt init` to create a template file and see what can be modified.
 
 ## Using in Python

pyflowbatt/analysis.py

@@ -6,6 +6,7 @@
 import re
 import uuid
 import zipfile
+from collections.abc import Sequence
 from pathlib import Path, PurePosixPath
 from typing import Literal
 from urllib.parse import unquote
@@ -1293,7 +1294,7 @@ def _merged_row_order(indexes: list[pd.Index]) -> list[str]:
 
 
 def merge_summaries(
-    summary_xlsxs: list[str | Path],
+    summary_xlsxs: Sequence[str | Path],
     sample_ids_by_sample_folder: dict[Path, str] | None = None,
 ) -> pd.DataFrame:
     """Merge all summary sheets into one mega summary."""
@@ -1326,6 +1327,73 @@ def merge_summaries(
     return df[cols]
 
 
+def merge_folder_summaries(
+    summaries_by_folder: dict[str, list[Path]],
+    sample_ids_by_sample_folder: dict[Path, str] | None = None,
+) -> pd.DataFrame:
+    """Each folder's Average and Std over its samples, side by side."""
+    merged = {
+        label: merge_summaries(paths, sample_ids_by_sample_folder).set_index("Quantity")
+        for label, paths in summaries_by_folder.items()
+    }
+    df = pd.DataFrame(
+        {f"{label} {stat}": m[stat] for label, m in merged.items() for stat in ("Average", "Std")}
+    )
+    order = _merged_row_order([m.index for m in merged.values()])
+    return df.reindex(order).rename_axis("Quantity").reset_index()
+
+
+def write_summary_xlsx(path: Path, sheets: dict[str, pd.DataFrame]) -> None:
+    """Write each table to its own autofitted sheet."""
+    with pd.ExcelWriter(path, engine="xlsxwriter") as writer:
+        for name, df in sheets.items():
+            df.to_excel(writer, index=False, sheet_name=name)
+            writer.sheets[name].autofit()
+
+
+def folder_labels(folders: list[Path]) -> dict[Path, str]:
+    """Label each folder by its name, with as many parent folders as needed to tell them apart."""
+    depth = dict.fromkeys(folders, 1)
+    while True:
+        labels = {f: Path(*f.parts[-depth[f] :]).as_posix() for f in folders}
+        clashes = [
+            f
+            for f in folders
+            if list(labels.values()).count(labels[f]) > 1 and depth[f] < len(f.parts)
+        ]
+        if not clashes:
+            return labels
+        for f in clashes:
+            depth[f] += 1
+
+
+def write_multi_folder_summary(
+    sample_ids_by_folder: dict[Path, dict[Path, str]], path: Path
+) -> bool:
+    """Write all samples' summaries, and each folder's averages, from several folders into path.
+
+    Returns False if no folder has any sample summaries.
+    """
+    sample_ids = {sf: sid for ids in sample_ids_by_folder.values() for sf, sid in ids.items()}
+    labels = folder_labels(list(sample_ids_by_folder))
+    summaries_by_folder = {
+        labels[folder]: summaries
+        for folder, ids in sample_ids_by_folder.items()
+        if (summaries := [s for sf in ids if (s := sf / "results" / "summary.xlsx").exists()])
+    }
+    if not summaries_by_folder:
+        return False
+    all_summaries = [s for summaries in summaries_by_folder.values() for s in summaries]
+    write_summary_xlsx(
+        path,
+        {
+            "Summary": merge_summaries(all_summaries, sample_ids),
+            "By folder": merge_folder_summaries(summaries_by_folder, sample_ids),
+        },
+    )
+    return True
+
+
 def analyse_all_samples(  # noqa: PLR0913
     folder: str | Path,
     *,
@@ -1334,8 +1402,8 @@ def analyse_all_samples(  # noqa: PLR0913
     max_folder_searches: int = DEFAULT_SEARCH,
     zip_output: bool = False,
     reporter: Reporter | None = None,
-) -> None:
-    """Take a folder and run all analysis.
+) -> dict[Path, str]:
+    """Take a folder and run all analysis, returns the sample ID of each sample folder.
 
     If ``zip_output`` is True, everything is zipped into a single zip file next to
     ``folder`` once analysis finishes, for uploading to Zenodo. The zip holds an RO-Crate
@@ -1356,7 +1424,7 @@ def analyse_all_samples(  # noqa: PLR0913
     )
     if len(sample_folders) == 0:
         logger.error("No sample folders found in %s", folder)
-        return
+        return {}
     configs_by_sample_folder: dict[Path, PyFlowBattConfig] = {
         sample_folder: PyFlowBattConfig.load(sample_folder) for sample_folder in sample_folders
     }
@@ -1401,12 +1469,7 @@ def analyse_all_samples(  # noqa: PLR0913
                     folder, max_search_depth, max_folder_searches, search_config
                 )
                 df = merge_summaries(summaries, sample_ids_by_sample_folder)
-                writer = pd.ExcelWriter(folder / "combined_summary.xlsx", engine="xlsxwriter")
-                df.to_excel(writer, index=False, sheet_name="Summary")
-                workbook = writer.book
-                worksheet = writer.sheets["Summary"]
-                worksheet.autofit()
-                workbook.close()
+                write_summary_xlsx(folder / "combined_summary.xlsx", {"Summary": df})
                 logger.debug("Combined all the results into one big summary")
 
         if zip_output:
@@ -1454,6 +1517,7 @@ def analyse_all_samples(  # noqa: PLR0913
                 crate_path.unlink(missing_ok=True)
 
     reporter.finish()
+    return sample_ids_by_sample_folder
 
 
 def dry_analyse_all_samples(
@@ -1504,7 +1568,7 @@ def dry_analyse_all_samples(
     logger.info("Results would be saved as battery data format in '%s' files.", save_format)
     if len(sample_folders) > 1:
         logger.info(
-            "Then I would combine all the summaries into one 'combined_results.xlsx' "
+            "Then I would combine all the summaries into one 'combined_summary.xlsx' "
             "at the root '%s'.",
             folder.name,
         )

pyflowbatt/cli.py

@@ -70,8 +70,8 @@ def analyse(  # noqa: PLR0913
     max_folder_searches: int = 10000,
     max_search_depth: int = 6,
     verbose: bool = False,
-) -> None:
-    """Run PyFlowBatt on a folder."""
+) -> dict[Path, str]:
+    """Run PyFlowBatt on a folder, returns the sample ID of each analysed sample folder."""
     import matplotlib as mpl  # noqa: PLC0415
 
     # Plots are only saved, and a GUI backend breaks when the live display's thread collects them
@@ -84,7 +84,7 @@ def analyse(  # noqa: PLR0913
     if not folderpath.is_dir():
         with _console_logging(reporter):
             logger.error("%s is not a directory", folderpath)
-        return
+        return {}
     log_path = None if dry else folderpath / LOG_FILENAME
     with _console_logging(reporter, verbose=verbose, log_path=log_path):
         try:
@@ -97,7 +97,7 @@ def analyse(  # noqa: PLR0913
                     zip_output=zip_output,
                 )
             else:
-                analyse_all_samples(
+                return analyse_all_samples(
                     folderpath,
                     max_folder_searches=max_folder_searches,
                     max_search_depth=max_search_depth,
@@ -107,6 +107,92 @@ def analyse(  # noqa: PLR0913
         except ConfigError as e:
             # The message names the file and line, a traceback would only hide it
             logger.error("%s", e)  # noqa: TRY400
+    return {}
+
+
+def _combined_summary_path(path: str, *, dry: bool) -> Path:
+    """Resolve --combined-summary to an .xlsx file, raising ValueError if it can't be written."""
+    from pyflowbatt.analysis import check_path_writable  # noqa: PLC0415
+
+    summary_path = Path(path).resolve()
+    if summary_path.is_dir():
+        summary_path /= "combined_summary.xlsx"
+    elif summary_path.suffix.lower() != ".xlsx":
+        msg = f"--combined-summary must be a folder or an .xlsx file, not {path}"
+        raise ValueError(msg)
+    if not summary_path.parent.is_dir():
+        msg = f"Cannot write {summary_path.name}: {summary_path.parent} is not a directory"
+        raise ValueError(msg)
+    if not dry:
+        try:
+            check_path_writable(summary_path)
+        except PermissionError as e:
+            raise ValueError(str(e)) from e
+    return summary_path
+
+
+def analyse_folders(  # noqa: PLR0913
+    folders: list[str],
+    *,
+    combined_summary: str | None = None,
+    dry: bool = False,
+    zip_output: bool = False,
+    max_folder_searches: int = 10000,
+    max_search_depth: int = 6,
+    verbose: bool = False,
+) -> None:
+    """Run PyFlowBatt on each folder, then combine all their summaries if asked to."""
+    from pyflowbatt.analysis import write_multi_folder_summary  # noqa: PLC0415
+
+    paths = list(dict.fromkeys(Path(f).resolve() for f in folders))
+    with _console_logging(RichReporter()):
+        if bad := [p for p in paths if not p.is_dir()]:
+            logger.error("Not a directory: %s", ", ".join(map(str, bad)))
+            return
+        if nested := [(a, b) for a in paths for b in paths if a != b and b.is_relative_to(a)]:
+            # Samples in both would be analysed twice and counted twice in the averages
+            logger.error("%s is inside %s, pass only one of them", *nested[0][::-1])
+            return
+        try:
+            summary_path = (
+                _combined_summary_path(combined_summary, dry=dry) if combined_summary else None
+            )
+        except ValueError as e:
+            logger.error("%s", e)  # noqa: TRY400
+            return
+
+    sample_ids_by_folder = {
+        path: analyse(
+            str(path),
+            dry=dry,
+            zip_output=zip_output,
+            max_folder_searches=max_folder_searches,
+            max_search_depth=max_search_depth,
+            verbose=verbose,
+        )
+        for path in paths
+    }
+    if summary_path is None:
+        return
+
+    reporter = RichReporter(verbose=verbose)
+    with _console_logging(reporter, verbose=verbose):
+        if dry:
+            logger.info(
+                "Then I would combine the summaries from all %d folders into '%s'.",
+                len(paths),
+                summary_path,
+            )
+            return
+        with (
+            reporter.section(f"Across {len(paths)} folders"),
+            reporter.task("Combined summary", detail=summary_path.name),
+        ):
+            written = write_multi_folder_summary(sample_ids_by_folder, summary_path)
+            if not written:
+                logger.error("No sample summaries to combine")
+        if written:
+            logger.info("\nCombined summary of all folders: %s", summary_path)
 
 
 def main() -> None:
@@ -131,7 +217,22 @@ def main() -> None:
         "--dry", action="store_true", help="Don't do any analysis, just check the folders"
     )
     parser.add_argument(
-        "--folder", type=str, help="Folder to analyse, if not specified, analyse current folder"
+        "--folder",
+        type=str,
+        nargs="+",
+        help=(
+            "Folder(s) to analyse, if not specified, analyse current folder. "
+            "Several folders are analysed one after another."
+        ),
+    )
+    parser.add_argument(
+        "--combined-summary",
+        type=str,
+        metavar="PATH",
+        help=(
+            "Also combine the summaries of every folder analysed into one Excel file at PATH, "
+            "an .xlsx file or a folder to write combined_summary.xlsx into"
+        ),
     )
     parser.add_argument(
         "--zip",
@@ -169,7 +270,12 @@ def main() -> None:
         kwargs["max_folder_searches"] = args.max_folder_searches
     if args.max_search_depth is not None:
         kwargs["max_search_depth"] = args.max_search_depth
-    analyse(folder=args.folder, dry=args.dry, zip_output=args.zip, verbose=args.verbose, **kwargs)
+    kwargs.update(dry=args.dry, zip_output=args.zip, verbose=args.verbose)
+    if args.combined_summary or (args.folder and len(args.folder) > 1):
+        folders = args.folder or [str(Path.cwd())]
+        analyse_folders(folders, combined_summary=args.combined_summary, **kwargs)
+    else:
+        analyse(folder=args.folder[0] if args.folder else None, **kwargs)
 
 
 if __name__ == "__main__":