Rebuild ExcelTableCNN: working TableSense-inspired detector with PBR, grid-context backbone, and measured results (v0.3.0)

.gitignore

@@ -123,3 +123,5 @@ data/*
 !data/README.md
 code_snippets.py
 .venv
+docs/*
+*.pt

archive/README.md

@@ -0,0 +1,12 @@
+# Archive
+
+Unmaintained code kept for reference only. Nothing in this folder is part of
+the `excel_table_cnn` package, is covered by tests, or is expected to run.
+
+| Folder / file | What it was | Why it's here |
+|---|---|---|
+| `experimental_model/` | A from-scratch Faster-R-CNN-style detector (hand-written anchor generator, RPN heads, RoIAlign, NMS, detection head with a PBR output). | Superseded by the torchvision-based model in `excel_table_cnn/model/`. It could never learn as written — it computes no RPN loss, so proposals stay random and the loss collapses to ~0. Kept because its PBR head structure is the starting point for the planned paper-faithful PBR module (see the project roadmap). |
+| `ml_classification/`, `algorithms/`, `table_detector.py` | A pre-CNN prototype: sklearn per-cell header classifier plus heuristic table-body growing. | Different approach and scope; depends on a private library (`dkslib`) and broken import paths. Kept for historical reference. |
+| `excel-cnn-testing.ipynb` | The old Kaggle driver notebook. | Its imports point at module paths that no longer exist. Replaced by `notebooks/train_kaggle_colab.ipynb`. |
+| `dl_tensors.ipynb` | Early tensor-building exploration for the header classifier. | Obsolete. |
+| `code_snippets.py` | Scratchpad of torchvision API experiments. | Never runnable; reference only. Gitignored (local copy only). |

excel_table_cnn/__init__.py

@@ -0,0 +1,56 @@
+"""ExcelTableCNN: spreadsheet table detection with CNNs.
+
+An independent open-source reimplementation inspired by the TableSense paper
+(Dong et al., AAAI 2019, arXiv:2106.13500).
+"""
+
+__version__ = "0.3.0"
+
+from .data.census import box_census, lattice_coverage
+from .data.features import FEATURE_NAMES, NUM_FEATURES, featurize_sheet
+from .data.pipeline import build_samples, build_sheet_sample, get_train_test
+from .data.workbook import UnsupportedFormatError, WorkbookReader, load_sheet_array
+from .device import resolve_device
+from .evaluation.eob import eob, eob_precision_recall
+from .evaluation.evaluate import evaluate_model, format_report
+from .inference import detect_tables, load_model
+from .model.detector import TableDetectionModel, build_model
+from .training.dataset import (
+    SpreadsheetDataset,
+    box_to_range,
+    collate_fn,
+    parse_table_range,
+)
+from .training.train import TrainConfig, load_checkpoint, save_checkpoint, train_model
+
+__all__ = [
+    "__version__",
+    "box_census",
+    "lattice_coverage",
+    "FEATURE_NAMES",
+    "NUM_FEATURES",
+    "featurize_sheet",
+    "build_samples",
+    "build_sheet_sample",
+    "get_train_test",
+    "UnsupportedFormatError",
+    "WorkbookReader",
+    "load_sheet_array",
+    "resolve_device",
+    "eob",
+    "eob_precision_recall",
+    "evaluate_model",
+    "format_report",
+    "detect_tables",
+    "load_model",
+    "TableDetectionModel",
+    "build_model",
+    "SpreadsheetDataset",
+    "box_to_range",
+    "collate_fn",
+    "parse_table_range",
+    "TrainConfig",
+    "load_checkpoint",
+    "save_checkpoint",
+    "train_model",
+]

excel_table_cnn/data/census.py

@@ -0,0 +1,77 @@
+"""Ground-truth box census: measure table shapes and anchor-lattice coverage.
+
+Used to derive the anchor defaults in ``model/rcnn.py`` from the actual
+annotation corpus instead of guessing. Rerun on a new corpus with::
+
+    from excel_table_cnn.data.census import box_census, lattice_coverage
+    from excel_table_cnn.data.markup import MarkupLoader
+    stats = box_census(MarkupLoader().get_markup("tablesense"))
+"""
+
+import math
+from typing import Dict, List, Sequence, Tuple
+
+from ..training.dataset import parse_table_range
+
+
+def collect_box_dims(markup_df) -> List[Tuple[float, float]]:
+    """(width, height) in cells for every parseable annotated table range."""
+    dims: List[Tuple[float, float]] = []
+    for ranges in markup_df["table_range"]:
+        for rng in ranges:
+            rng = rng.strip()
+            if not rng:
+                continue
+            try:
+                x1, y1, x2, y2 = parse_table_range(rng)
+            except ValueError:
+                continue
+            dims.append((x2 - x1, y2 - y1))
+    return dims
+
+
+def _percentile(sorted_values: Sequence[float], p: float) -> float:
+    return sorted_values[round(p / 100 * (len(sorted_values) - 1))]
+
+
+def box_census(markup_df) -> Dict[str, Dict[str, float]]:
+    """Percentile summary of widths, heights, sqrt-areas and h/w ratios."""
+    dims = collect_box_dims(markup_df)
+    series = {
+        "width": sorted(w for w, _ in dims),
+        "height": sorted(h for _, h in dims),
+        "sqrt_area": sorted(math.sqrt(w * h) for w, h in dims),
+        "hw_ratio": sorted(h / w for w, h in dims),
+    }
+    return {
+        name: {f"p{p}": _percentile(values, p) for p in (1, 5, 25, 50, 75, 95, 99)}
+        for name, values in series.items()
+    }
+
+
+def _anchor_dims(size: float, ratio: float) -> Tuple[float, float]:
+    """torchvision convention: h = size*sqrt(ratio), w = size/sqrt(ratio)."""
+    return size / math.sqrt(ratio), size * math.sqrt(ratio)
+
+
+def lattice_coverage(
+    dims: Sequence[Tuple[float, float]],
+    sizes: Sequence[float],
+    ratios: Sequence[float],
+    iou_thresholds: Sequence[float] = (0.5, 0.7),
+) -> Dict[float, float]:
+    """Fraction of GT boxes whose best (centered) anchor reaches each IoU."""
+
+    def best_iou(width: float, height: float) -> float:
+        best = 0.0
+        for size in sizes:
+            for ratio in ratios:
+                aw, ah = _anchor_dims(size, ratio)
+                inter = min(width, aw) * min(height, ah)
+                union = width * height + aw * ah - inter
+                best = max(best, inter / union)
+        return best
+
+    coverage = [best_iou(w, h) for w, h in dims]
+    n = max(len(coverage), 1)
+    return {t: sum(c >= t for c in coverage) / n for t in iou_thresholds}

excel_table_cnn/data/converter.py

@@ -0,0 +1,50 @@
+"""Convert legacy spreadsheet formats to .xlsx via headless LibreOffice.
+
+Openpyxl only reads .xlsx; the VEnron2 corpus ships as .xls. Conversion is
+lossy for some formatting details, which is a known limitation — see the
+README's dataset section.
+"""
+
+import logging
+import os
+import subprocess
+
+from tqdm import tqdm
+
+logger = logging.getLogger(__name__)
+
+CONVERTIBLE_EXTENSIONS = (".xls", ".xlsb")
+
+
+def convert_file(file_path: str, output_dir: str) -> bool:
+    """Convert one file to .xlsx next to it. Returns True if the .xlsx exists."""
+    output_file_path = os.path.splitext(file_path)[0] + ".xlsx"
+    if os.path.exists(output_file_path):
+        return True
+    try:
+        subprocess.run(
+            ["libreoffice", "--headless", "--convert-to", "xlsx",
+             file_path, "--outdir", output_dir],
+            check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+        )
+    except (subprocess.CalledProcessError, FileNotFoundError) as exc:
+        logger.error("Error converting %s: %s", file_path, exc)
+    return os.path.exists(output_file_path)
+
+
+def convert_files(files_df, data_folder_path: str):
+    """Convert every .xls/.xlsb row of a files dataframe; returns an updated
+    copy where converted rows point at the .xlsx file name."""
+    updated_files_df = files_df.copy()
+
+    for index, row in tqdm(files_df.iterrows(), total=files_df.shape[0],
+                           desc="Converting files"):
+        file_name, file_ext = os.path.splitext(row["file_name"])
+        if file_ext.lower() not in CONVERTIBLE_EXTENSIONS:
+            continue
+        file_path = os.path.join(data_folder_path, row["parent_path"], row["file_name"])
+        output_directory = os.path.join(data_folder_path, row["parent_path"])
+        if convert_file(file_path, output_directory):
+            updated_files_df.at[index, "file_name"] = file_name + ".xlsx"
+
+    return updated_files_df