diff --git a/backend/src/Tools/MathComps.Cli.Handouts/BuildCommand.cs b/backend/src/Tools/MathComps.Cli.Handouts/BuildCommand.cs index e2e30a8..8378db4 100644 --- a/backend/src/Tools/MathComps.Cli.Handouts/BuildCommand.cs +++ b/backend/src/Tools/MathComps.Cli.Handouts/BuildCommand.cs @@ -192,6 +192,20 @@ public override async Task ExecuteAsync(CommandContext context, Settings se // variant would re-run the same asy+inkscape pipeline. var asyAlreadyRecompiled = new HashSet(StringComparer.OrdinalIgnoreCase); + // R2 keys already pushed in this run — language variants share image keys, so the + // second variant of a handout skips re-uploading what the first variant just sent. + var uploadedR2Keys = new HashSet(StringComparer.Ordinal); + + // Persistent ledger mapping each R2 key to the SVG mtime that was last pushed under it. + // An image is fresh on R2 exactly when its current mtime is not newer than the value + // recorded here, regardless of how/when it was generated (asy pipeline inside the build, + // manual `asy` compile beforehand, AI invoking the export script directly, partial runs + // that only touched a subset of handouts). Missing entry ⇒ never uploaded ⇒ must push. + var uploadStatePath = Path.Combine(inputDirectory.FullName, ".r2-uploads.json"); + var uploadState = File.Exists(uploadStatePath) + ? File.ReadAllText(uploadStatePath).FromJson>() + : []; + // Process each discovered handout file. foreach (var inputFile in inputFiles) { @@ -214,8 +228,10 @@ public override async Task ExecuteAsync(CommandContext context, Settings se else AnsiConsole.MarkupLine(" [yellow]⚠ Asy compile skipped (--skip-asy)[/]"); - // Generate the skeleton and compile both TeX files. + // Prepare the skeleton file info, should we generate it FileInfo? skeletonFile = null; + + // If we're actuallly compiling if (!settings.SkipCompile) { // Generate the skeleton .tex @@ -262,13 +278,45 @@ public override async Task ExecuteAsync(CommandContext context, Settings se // Upload images and PDFs to R2 (unless skipped) if (uploader is not null) { - // Upload queued images, logging each so the publish run shows what landed in R2 - foreach (var upload in pendingImageUploads) + // Partition the queued image uploads: drop ones already pushed in this run + // (a sibling language variant covered the same R2 key) and ones whose current + // on-disk SVG mtime matches the value recorded the last time we uploaded under + // that key (⇒ same bytes already on R2). What remains is the set of SVGs that + // are either brand-new (no ledger entry) or have been rewritten on disk since + // their last successful push. + var uploadsToSend = pendingImageUploads + .Where(upload => !uploadedR2Keys.Contains(upload.R2Key)) + .Where(upload => !uploadState.TryGetValue(upload.R2Key, out var lastMtime) + || File.GetLastWriteTimeUtc(upload.SourcePath) > lastMtime) + .ToList(); + + // How many we filtered out — surfaced in the summary line below. + var skippedUploadCount = pendingImageUploads.Count - uploadsToSend.Count; + + // Handle the uploads sequentially + foreach (var upload in uploadsToSend) { + // Capture the SVG mtime up front — this is the version we're about to push + // and want to record as "the bytes R2 has now" once the upload succeeds. + var svgMtime = File.GetLastWriteTimeUtc(upload.SourcePath); + + // Push the image await uploader.UploadAsync(upload.SourcePath, upload.R2Key); + + // Mark this R2 key as handled so language variants later in this run skip it... + uploadedR2Keys.Add(upload.R2Key); + + // ..And persist the mtime we just pushed so the NEXT run knows R2 has this version + uploadState[upload.R2Key] = svgMtime; + + // Per-image success log AnsiConsole.MarkupLine($" [green]✓ Image uploaded:[/] {Markup.Escape(Path.GetFileName(upload.SourcePath))}"); } + // Summary line so the publish run shows that unchanged images were intentionally skipped. + if (skippedUploadCount > 0) + AnsiConsole.MarkupLine($" [dim]· {skippedUploadCount} image(s) unchanged, skipped upload[/]"); + // PDFs only when we actually compiled them; otherwise the on-disk copies are // from an earlier run and we'd risk publishing binaries that don't match the source. if (skeletonFile is not null) @@ -293,6 +341,18 @@ public override async Task ExecuteAsync(CommandContext context, Settings se // All handout files processed + // If we did any uploads, update the upload state file + if (uploader is not null) + { + // Prepare the sorted ledges, sorted so the file is human-readable for the occasional manual inspection. + var sortedState = uploadState + .OrderBy(entry => entry.Key, StringComparer.Ordinal) + .ToDictionary(entry => entry.Key, entry => entry.Value); + + // Persist the upload ledger + File.WriteAllText(uploadStatePath, sortedState.ToJson()); + } + // Report unknown commands if any were found. if (allUnknownCommands.Count != 0) { diff --git a/backend/src/Tools/MathComps.Cli.Handouts/README.md b/backend/src/Tools/MathComps.Cli.Handouts/README.md index bbf9b01..3490916 100644 --- a/backend/src/Tools/MathComps.Cli.Handouts/README.md +++ b/backend/src/Tools/MathComps.Cli.Handouts/README.md @@ -6,11 +6,12 @@ A .NET tool that orchestrates the full handout build pipeline: generates skeleto For each matched `.tex` file, the tool runs these steps in order: -1. **Generate skeleton** — strips solutions/proofs/hints, produces a `-skeleton.tex` worksheet -2. **Compile TeX** — runs the configured compiler (2 passes) on both main + skeleton files -3. **Parse to JSON** — converts the TeX document structure into `RawContentBlock[]` JSON (saved locally to `web/src/content/handouts/`) -4. **Upload images** — processes SVG images and uploads them to R2 under `handouts//.svg`, where `` is the language-stripped handout id (so all language variants share one image set) -5. **Upload PDFs** — uploads compiled main + skeleton PDFs to R2 under `handouts/pdfs/.pdf` (flat layout; every handout's PDFs share one folder) +1. **Refresh figures** — for every `.asy`-backed image the handout references, checks whether the compiled PDF/SVG are stale relative to the `.asy` source plus its transitive `include`/`import` deps, and batch-recompiles the stale ones via `Images/_Export-Asy.ps1` (asy → PDF, then Inkscape PDF → SVG). See [Image pipeline](#image-pipeline-asymptote) below. +2. **Generate skeleton** — strips solutions/proofs/hints, produces a `-skeleton.tex` worksheet +3. **Compile TeX** — runs the configured compiler (2 passes) on both main + skeleton files +4. **Parse to JSON** — converts the TeX document structure into `RawContentBlock[]` JSON (saved locally to `web/src/content/handouts/`) +5. **Upload images** — pushes SVGs to R2 under `handouts//.svg`, where `` is the language-stripped handout id (so all language variants share one image set). Only SVGs whose on-disk mtime differs from the value recorded in `data/handouts/.r2-uploads.json` are pushed — unchanged figures are skipped. +6. **Upload PDFs** — uploads compiled main + skeleton PDFs to R2 under `handouts/pdfs/.pdf` (flat layout; every handout's PDFs share one folder) ## Prerequisites @@ -62,6 +63,22 @@ dotnet run -- --skip-compile *.sk.tex dotnet run -- --skip-upload *.sk.tex ``` +### Skip the Asymptote Figure Refresh + +For runs where you trust the on-disk figures (e.g. CI without an Asymptote toolchain), or to shave the per-handout dep scan: + +```bash +dotnet run -- --skip-asy *.sk.tex +``` + +### Force-Recompile Every Figure + +Use after editing `Images/_common.asy` in a way that changes how existing figures render (palette tweak, modified helper). Such edits are deliberately not tracked by the dep graph — see [Image pipeline](#image-pipeline-asymptote): + +```bash +dotnet run -- --force-asy *.sk.tex +``` + ### Custom Compiler ```bash @@ -70,12 +87,38 @@ dotnet run -- --compiler pdfcsplain *.sk.tex ## Options -| Option | Default | Description | -| ---------------- | ------------ | ------------------------------------------------------- | -| `` | required | File pattern(s) to match (e.g. `*.sk.tex`) | -| `--compiler` | `pdfcsplain` | TeX compiler command | -| `--skip-compile` | `false` | Skip TeX compilation, only parse and copy existing PDFs | -| `--skip-upload` | `false` | Skip uploading PDFs and images to R2 | +| Option | Default | Description | +| ---------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `` | required | File pattern(s) to match (e.g. `*.sk.tex`) | +| `--compiler` | `pdfcsplain -interaction=nonstopmode -halt-on-error` | TeX compiler command (full string, including flags) | +| `--skip-compile` | `false` | Skip TeX compilation, only parse and upload existing PDFs | +| `--skip-upload` | `false` | Skip uploading PDFs and images to R2 | +| `--skip-asy` | `false` | Skip the Asymptote staleness check + recompilation entirely | +| `--force-asy` | `false` | Recompile every `.asy`-backed figure regardless of staleness (used after a semantic `_common.asy` edit) | +| `--error-log` | `errors.log` | Path to the error log appended to on compiler failure | + +## Image pipeline (Asymptote) + +Figures live in `data/handouts/Images/` as `.asy` sources that compile to `.pdf` (consumed by the TeX engine) and `.svg` (consumed by the web frontend). The build wraps two layers around this: + +### Staleness check + +Before touching the TeX pipeline, the build walks every image the document references and decides per figure whether to recompile: + +- If the figure has no sibling `.asy` (externally authored raster/PDF), it's left alone. +- Otherwise the build resolves the figure's transitive dependency set — its own source plus every file pulled in via `import ;` or `include "";` — and recompiles when any source is newer than the older of the two compiled outputs (`.pdf` or `.svg` missing also counts as stale). +- `Images/_common.asy` is intentionally **excluded** from the dep graph. Most edits to `_common.asy` are additive helpers that can't affect existing figures, so cascading invalidation across every figure would be pure waste. When a `_common.asy` change *does* alter rendering (palette tweak, modified helper used by existing figures), opt in with `--force-asy`. + +Stale figures are batched into a single invocation of `Images/_Export-Asy.ps1` per run, which handles the `asy → PDF` render and the Inkscape `PDF → SVG` conversion. + +### Upload ledger + +R2 uploads are gated by `data/handouts/.r2-uploads.json` (gitignored). It maps each R2 key to the SVG mtime that was last successfully pushed under that key. On every run: + +- An image is pushed only when its current on-disk mtime is newer than the recorded value (or no entry exists). +- After each successful upload the ledger is updated and persisted at the end of the run. + +This works the same regardless of how the SVG came to be on disk — pipeline-recompiled, `--force-asy`'d, hand-rendered with `asy`, or generated by another tool. Wiping the ledger forces a fresh upload of everything. ## Deployment Workflow