From 5ccc38478e2ac9769eb2add41bd567f4caecb35e Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 20 Feb 2026 19:52:38 +0000
Subject: [PATCH 1/2] Initial plan


From 977b4499d5e6203c12ea4117bf3849c92753e7d1 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 20 Feb 2026 19:58:35 +0000
Subject: [PATCH 2/2] Fix inefficient file loading: avoid double-reading
 parquet files in load_samples and remove_empty_parquets

Co-authored-by: dprim7 <38478631+dprim7@users.noreply.github.com>
---
 src/HH4b/utils.py | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/src/HH4b/utils.py b/src/HH4b/utils.py
index e4f0f5de..7331b843 100644
--- a/src/HH4b/utils.py
+++ b/src/HH4b/utils.py
@@ -112,7 +112,7 @@ def remove_empty_parquets(samples_dir, year):
         parquet_files = listdir(f"{samples_dir}/{year}/{sample}/parquet")
         for f in parquet_files:
             file_path = f"{samples_dir}/{year}/{sample}/parquet/{f}"
-            if not len(pd.read_parquet(file_path)):
+            if not len(pd.read_parquet(file_path, columns=[])):
                 print("Removing: ", f"{sample}/{f}")
                 Path(file_path).unlink()
 
@@ -367,24 +367,16 @@ def load_samples(
             try:
                 non_empty_passed_list = []
                 for parquet_file in parquet_path.glob("*.parquet"):
-                    if not pd.read_parquet(parquet_file).empty:
-                        df_sample = pd.read_parquet(
-                            parquet_file, filters=filters, columns=load_columns
-                        )
+                    df_sample = pd.read_parquet(
+                        parquet_file, filters=filters, columns=load_columns
+                    )
+                    if not df_sample.empty:
                         non_empty_passed_list.append(df_sample)
                 events = pd.concat(non_empty_passed_list, ignore_index=True)
             except Exception:
                 warnings.warn(
                     f"Can't read file with requested columns/filters for {sample}!", stacklevel=1
                 )
-                non_empty_passed_list = []
-                for parquet_file in parquet_path.glob("*.parquet"):
-                    if not pd.read_parquet(parquet_file).empty:
-                        df_sample = pd.read_parquet(
-                            parquet_file, filters=filters, columns=load_columns
-                        )
-                        non_empty_passed_list.append(df_sample)
-                events = pd.concat(non_empty_passed_list, ignore_index=True)
                 continue
 
             # no events?