First-reading review and ANU change diffs

apps/web/lib/catalogue-import/kinds/course/prompt.ts

@@ -49,7 +49,7 @@ Requisites:
 - Model the whole rule whenever the page's punctuation settles its grouping. Use unmodelledText, with a review item, only for wording you genuinely cannot place in the rule.
 
 Evidence and review:
-- Give evidence for each field you fill. Its fieldKey is the exact field path, such as requisites.prerequisiteRule or offerings.
+- Give evidence for every field you fill, not only tags and requisites: title, description, unit value, offerings, fees, assessment, learning outcomes, areas of interest and the rest each get an entry. Its fieldKey is the exact field path, such as title, requisites.prerequisiteRule or offerings. A field without evidence reaches the reviewer with no confidence.
 - Confidence is how directly the page states the value, from 0 to 1.
 - Add specific review items for ambiguity, unsupported wording or conflicting statements on the page.
 - Do not include chain-of-thought, hidden reasoning, commentary or self-evaluation. Only return the schema fields.`;

apps/web/lib/catalogue-sync/persist-source-version.ts

@@ -14,7 +14,10 @@ import {
   contentHashForCatalogueContent,
   readVersionContent,
 } from "../catalogue-import/version-content.ts";
-import { generateSourceReview } from "../catalogue/source-review-store.ts";
+import {
+  generateFirstReadReview,
+  generateSourceReview,
+} from "../catalogue/source-review-store.ts";
 
 export type PersistedSourceVersion = {
   status: "unchanged" | "review_required" | "applied";
@@ -545,16 +548,85 @@ export async function persistSourceVersion(
         populatedDraft: Boolean(draftFromSource),
       };
     }
+    const lockDraft = async () => {
+      const [row] =
+        await tx`select content, content_hash from public.catalogue_drafts
+        where record_id = ${claim.recordId} for update`;
+      return row;
+    };
+    const hasMeaningfulLocalContent = (
+      draft: Record<string, unknown> | undefined,
+    ) => {
+      const empty = emptyCatalogueContent({
+        kind: claim.kind,
+        code: claim.code,
+        academicYear: claim.academicYear,
+        title:
+          record.listing_title === null ? null : String(record.listing_title),
+      });
+      return (
+        record.published_version_id !== null ||
+        (draft !== undefined &&
+          String(draft.content_hash) !== contentHashForCatalogueContent(empty))
+      );
+    };
+    /**
+     * Fills the draft from a source version and queues every part of it for
+     * a person. Review rows belong to the sync that made the version, so
+     * they stay joined to it when this sync found nothing new.
+     */
+    const populateDraft = async (
+      sourceVersionId: number,
+      reviewSyncId: string,
+    ) => {
+      await tx`insert into public.catalogue_drafts (
+        record_id, base_version_id, content, content_hash, content_schema_version,
+        revision, updated_by
+      ) values (${claim.recordId}, ${sourceVersionId}, ${tx.json(write as never)},
+        ${write.contentHash}, ${CATALOGUE_CONTENT_SCHEMA_VERSION}, 0, null)
+      on conflict (record_id) do update set base_version_id = excluded.base_version_id,
+        content = excluded.content, content_hash = excluded.content_hash,
+        content_schema_version = excluded.content_schema_version,
+        revision = public.catalogue_drafts.revision + 1, updated_by = null,
+        updated_at = now()`;
+      await tx`delete from public.catalogue_draft_provenance where record_id = ${claim.recordId}`;
+      await tx`insert into public.catalogue_draft_provenance (
+        record_id, field_path, origin, source_version_id, source_evidence_id
+      ) select ${claim.recordId}, field_path, method, ${sourceVersionId}, id
+        from public.catalogue_version_provenance where version_id = ${sourceVersionId}`;
+      await tx`insert into public.catalogue_change_events (
+        record_id, draft_revision, event_kind, origin, version_id
+      ) select ${claim.recordId}, revision, 'source_draft_created', 'source', ${sourceVersionId}
+        from public.catalogue_drafts where record_id = ${claim.recordId}`;
+      // The draft took the model's reading whole, so every part of it is
+      // queued for a person, rated by how sure the reading is.
+      await generateFirstReadReview(tx, {
+        syncId: reviewSyncId,
+        recordId: claim.recordId,
+        content: write,
+      });
+    };
     const previousSourceVersionId =
       record.latest_source_version_id === null
         ? null
         : Number(record.latest_source_version_id);
     const [previous] = previousSourceVersionId
-      ? await tx`select content_hash from public.catalogue_versions where id = ${previousSourceVersionId}`
+      ? await tx`select content_hash, sync_id from public.catalogue_versions where id = ${previousSourceVersionId}`
       : [];
     if (previous && String(previous.content_hash) === write.contentHash) {
       await tx`update public.catalogue_records set source_checked_at = now()
         where id = ${claim.recordId}`;
+      // ANU has not changed, but a record whose draft was discarded still
+      // wants the reading back.
+      const draft = await lockDraft();
+      if (!hasMeaningfulLocalContent(draft) && previous.sync_id !== null) {
+        await populateDraft(previousSourceVersionId!, String(previous.sync_id));
+        return {
+          status: "applied",
+          sourceVersionId: previousSourceVersionId!,
+          populatedDraft: true,
+        };
+      }
       await tx`insert into public.catalogue_change_events (
         record_id, event_kind, origin, actor_id, version_id
       ) values (${claim.recordId}, 'source_checked', 'source', null, ${previousSourceVersionId})`;
@@ -595,42 +667,11 @@ export async function persistSourceVersion(
       latest_source_version_id = ${sourceVersionId}, source_checked_at = now()
       where id = ${claim.recordId}`;
 
-    const [draft] =
-      await tx`select content, content_hash from public.catalogue_drafts
-      where record_id = ${claim.recordId} for update`;
-    const empty = emptyCatalogueContent({
-      kind: claim.kind,
-      code: claim.code,
-      academicYear: claim.academicYear,
-      title:
-        record.listing_title === null ? null : String(record.listing_title),
-    });
-    const hasMeaningfulLocalContent =
-      record.published_version_id !== null ||
-      (draft &&
-        String(draft.content_hash) !== contentHashForCatalogueContent(empty));
-    const populateDraft =
-      previousSourceVersionId === null && !hasMeaningfulLocalContent;
-    if (populateDraft) {
-      await tx`insert into public.catalogue_drafts (
-        record_id, base_version_id, content, content_hash, content_schema_version,
-        revision, updated_by
-      ) values (${claim.recordId}, ${sourceVersionId}, ${tx.json(write as never)},
-        ${write.contentHash}, ${CATALOGUE_CONTENT_SCHEMA_VERSION}, 0, null)
-      on conflict (record_id) do update set base_version_id = excluded.base_version_id,
-        content = excluded.content, content_hash = excluded.content_hash,
-        content_schema_version = excluded.content_schema_version,
-        revision = public.catalogue_drafts.revision + 1, updated_by = null,
-        updated_at = now()`;
-      await tx`delete from public.catalogue_draft_provenance where record_id = ${claim.recordId}`;
-      await tx`insert into public.catalogue_draft_provenance (
-        record_id, field_path, origin, source_version_id, source_evidence_id
-      ) select ${claim.recordId}, field_path, method, ${sourceVersionId}, id
-        from public.catalogue_version_provenance where version_id = ${sourceVersionId}`;
-      await tx`insert into public.catalogue_change_events (
-        record_id, draft_revision, event_kind, origin, version_id
-      ) select ${claim.recordId}, revision, 'source_draft_created', 'source', ${sourceVersionId}
-        from public.catalogue_drafts where record_id = ${claim.recordId}`;
+    const draft = await lockDraft();
+    // A record with nothing of its own, never read or discarded since, takes
+    // the reading whole rather than comparing it with nothing.
+    if (!hasMeaningfulLocalContent(draft)) {
+      await populateDraft(sourceVersionId, claim.syncId);
       return { status: "applied", sourceVersionId, populatedDraft: true };
     }
 

apps/web/lib/catalogue/drafts.ts

@@ -7,6 +7,7 @@ import type {
 import { withSyncDatabaseClient } from "@/lib/catalogue-sync/sync-store";
 import { diffSnapshotWrites } from "@/lib/catalogue-import/changes";
 import { insertVersionContent } from "@/lib/catalogue-sync/persist-source-version";
+import { countBlockingFirstReads } from "@/lib/catalogue/source-review-store";
 import {
   contentHashForCatalogueContent,
   readVersionContent,
@@ -540,6 +541,14 @@ export async function publishCatalogueDraft({
       const draft = draftFromRow(row);
       if (draft.revision !== expectedRevision)
         throw new CatalogueDraftConflictError(draft.revision);
+      // A first reading from ANU is the model's word until a person has
+      // looked at the parts it was unsure of.
+      const blocking = await countBlockingFirstReads(tx, recordId);
+      if (blocking > 0)
+        throw new CatalogueDraftError(
+          `${blocking} ${blocking === 1 ? "part" : "parts"} of the first ANU reading ${blocking === 1 ? "needs" : "need"} review before publishing. Approve or correct ${blocking === 1 ? "it" : "them"} on the Changes tab.`,
+          "FIRST_READ_REVIEW",
+        );
       const publishedContent = record.published_version_id
         ? await readVersionContent(tx, Number(record.published_version_id))
         : null;

apps/web/lib/catalogue/first-read.ts

@@ -0,0 +1,156 @@
+import type { CatalogueContent } from "./content.ts";
+import {
+  type CatalogueReviewUnit,
+  catalogueReviewUnits,
+  evidenceBelongsToReviewUnit,
+  reviewUnitEvidence,
+} from "./review-units.ts";
+
+/**
+ * How much a first reading needs an administrator:
+ *
+ * - `needs_review`: the model was unsure, flagged an error, or wrote a rule
+ *   Coursemap cannot check. Publishing waits until each one is approved or
+ *   corrected.
+ * - `check`: probably right, but worth a look. Approve in bulk.
+ * - `accepted`: stated plainly on the page, or given no confidence at all.
+ *   Folded away, and reopenable.
+ */
+export type FirstReadBand = "needs_review" | "check" | "accepted";
+
+/** The reason on a reading an administrator put back up for review. */
+export const MARKED_FOR_REVIEW = "Marked for review by an administrator";
+
+/** Below this the reading is a guess, and publishing waits on it. */
+export const NEEDS_REVIEW_BELOW = 0.7;
+/** From here up, a reading with no flags is taken as read. */
+export const ACCEPTED_FROM = 0.9;
+
+/**
+ * Taken as read without a person: nothing flagged, and either read with full
+ * confidence or given no confidence to judge by. The draft already holds the
+ * value, so review leaves it out and lists it only among all fields.
+ */
+export function isCertainFirstRead(item: {
+  band: FirstReadBand | null;
+  confidence: number | null;
+  reason?: string | null;
+}) {
+  return (
+    item.band !== null &&
+    item.reason !== MARKED_FOR_REVIEW &&
+    item.band !== "needs_review" &&
+    (item.confidence === 1 || item.confidence === null)
+  );
+}
+
+export type FirstReadItem = {
+  fieldPath: string;
+  unitKind: CatalogueReviewUnit["unitKind"];
+  value: unknown;
+  /** The weakest confidence behind the value, or null with no evidence. */
+  confidence: number | null;
+  band: FirstReadBand;
+  /** Why it landed in its band, in words an administrator acts on. */
+  reason: string;
+};
+
+function isEmpty(value: unknown) {
+  if (value === null || value === undefined) return true;
+  if (typeof value === "string") return value.trim() === "";
+  if (Array.isArray(value)) return value.length === 0;
+  return false;
+}
+
+/**
+ * What the model's own confidence cannot show about a requirement rule:
+ * wording it could not place, conditions it left for review, and one
+ * sentence split into several conditions.
+ */
+function ruleConcerns(content: CatalogueContent, fieldPath: string) {
+  const [root, ruleKey] = fieldPath.split(".");
+  if (root !== "requirements" || !ruleKey)
+    return { concerns: [], lowest: null };
+  const conditions = content.requirements.conditions.filter(
+    (condition) => condition.ruleKey === ruleKey,
+  );
+  const concerns: string[] = [];
+  if (conditions.some((condition) => condition.kind === "other")) {
+    concerns.push("Part of the rule is free text Coursemap cannot check");
+  }
+  if (conditions.some((condition) => condition.reviewState === "review")) {
+    concerns.push("The importer marked part of the rule for review");
+  }
+  const sentences = conditions
+    .map((condition) => condition.sourceText?.trim())
+    .filter((text): text is string => Boolean(text));
+  if (new Set(sentences).size < sentences.length) {
+    concerns.push("One sentence was split into several conditions");
+  }
+  const confidences = conditions.map((condition) => condition.confidence);
+  return {
+    concerns,
+    lowest: confidences.length ? Math.min(...confidences) : null,
+  };
+}
+
+/**
+ * Every filled part of a record read from ANU for the first time, rated for
+ * how much it needs a person. The model's confidence is how directly the page
+ * states a value, which it tends to overrate, so its flags and Coursemap's own
+ * checks on requirement rules can only move an item towards review.
+ */
+export function classifyFirstRead(content: CatalogueContent): FirstReadItem[] {
+  return catalogueReviewUnits(content).flatMap((unit) => {
+    if (isEmpty(unit.value)) return [];
+    const confidences = reviewUnitEvidence(content, unit.fieldPath)
+      .map((entry) => entry.confidence)
+      .filter((confidence): confidence is number => confidence !== null);
+    const flags = content.flags.filter((flag) =>
+      evidenceBelongsToReviewUnit(unit.fieldPath, flag.fieldPath),
+    );
+    const rule = ruleConcerns(content, unit.fieldPath);
+    const candidates = [
+      ...confidences,
+      ...(rule.lowest === null ? [] : [rule.lowest]),
+    ];
+    const confidence = candidates.length ? Math.min(...candidates) : null;
+    const error = flags.find((flag) => flag.severity === "error");
+    const warning = flags.find((flag) => flag.severity === "warning");
+
+    let band: FirstReadBand;
+    let reason: string;
+    if (error) {
+      band = "needs_review";
+      reason = error.message;
+    } else if (rule.concerns.length) {
+      band = "needs_review";
+      reason = rule.concerns.join(". ");
+    } else if (confidence !== null && confidence < NEEDS_REVIEW_BELOW) {
+      band = "needs_review";
+      reason = "The page does not state this plainly";
+    } else if (warning) {
+      band = "check";
+      reason = warning.message;
+    } else if (confidence === null) {
+      band = "accepted";
+      reason = "No confidence was given, so it was taken as read";
+    } else if (confidence < ACCEPTED_FROM) {
+      band = "check";
+      reason = "Probably right, worth a look";
+    } else {
+      band = "accepted";
+      reason = "Stated plainly on the page";
+    }
+    return [
+      {
+        fieldPath: unit.fieldPath,
+        unitKind: unit.unitKind,
+        value: unit.value,
+        confidence,
+        band,
+        reason,
+      },
+    ];
+  });
+}