Course tags and explorer filters

apps/web/app/courses/(directory)/page.tsx

@@ -7,15 +7,21 @@ import { DataTableShell } from "@/ui/admin/catalogue-table/catalogue-table";
 import { FilterBar } from "@/ui/common/filter-bar";
 import {
   loadAcademicYearOptions,
+  loadCourseFilterOptions,
   loadPublishedCoursePage,
+  type CourseFilterOptions,
   type PublishedCoursePage,
 } from "@/lib/coursemap/published-courses";
 import { CourseDirectory } from "../course-directory";
 
 type CoursesSearchParams = {
   q?: string | string[];
+  subject?: string | string[];
   level?: string | string[];
   session?: string | string[];
+  college?: string | string[];
+  area?: string | string[];
+  tag?: string | string[];
   year?: string | string[];
   page?: string | string[];
 };
@@ -33,12 +39,25 @@ export default async function CoursesPage({
   const page = Math.max(1, Number(firstParam(params.page)) || 1);
   const query = firstParam(params.q).slice(0, 100);
   const levelParam = firstParam(params.level);
-  const level = ["1", "2", "3", "4"].includes(levelParam) ? levelParam : "";
+  // A digit is that level; a digit and + is that level or higher, which is
+  // how degree rules ask for units ("2000-level or higher").
+  const level = /^[1-9]\+?$/u.test(levelParam) ? levelParam : "";
   const sessionParam = firstParam(params.session);
   const session = ["Semester 1", "Semester 2"].includes(sessionParam)
     ? sessionParam
     : "";
-  const filters = { query, level, session };
+  const subjectParam = firstParam(params.subject).toUpperCase();
+  const subject = /^[A-Z]{4}$/u.test(subjectParam) ? subjectParam : "";
+  const college = firstParam(params.college).slice(0, 120);
+  const area = firstParam(params.area).slice(0, 120);
+  const tag = firstParam(params.tag).slice(0, 120);
+  const filters = { query, subject, level, session, college, area, tag };
+  let filterOptions: CourseFilterOptions = {
+    subjects: [],
+    colleges: [],
+    areas: [],
+    tags: [],
+  };
   let yearOptions: Awaited<ReturnType<typeof loadAcademicYearOptions>> = [];
   let selectedAcademicYear = new Date().getFullYear();
   let result: PublishedCoursePage = {
@@ -58,19 +77,26 @@ export default async function CoursesPage({
       : availableYears.has(currentYear)
         ? currentYear
         : (yearOptions[0]?.year ?? currentYear);
-    result = await loadPublishedCoursePage({
-      academicYear: selectedAcademicYear,
-      page,
-      filters,
-    });
+    [result, filterOptions] = await Promise.all([
+      loadPublishedCoursePage({
+        academicYear: selectedAcademicYear,
+        page,
+        filters,
+      }),
+      loadCourseFilterOptions(selectedAcademicYear),
+    ]);
   } catch {
     // Show an explicit outage state rather than an empty catalogue.
     catalogueUnavailable = true;
   }
   const paginationSearchParams = {
     q: query || undefined,
+    subject: subject || undefined,
     level: level || undefined,
     session: session || undefined,
+    college: college || undefined,
+    area: area || undefined,
+    tag: tag || undefined,
     year: String(selectedAcademicYear),
   };
 
@@ -124,6 +150,16 @@ export default async function CoursesPage({
                   : `${option.year} · No published courses`,
               })),
             },
+            {
+              key: "subject",
+              label: "Subject",
+              options: filterOptions.subjects.map((option) => ({
+                value: option.code,
+                label: option.name
+                  ? `${option.code} · ${option.name}`
+                  : option.code,
+              })),
+            },
             {
               key: "level",
               label: "Level",
@@ -132,6 +168,8 @@ export default async function CoursesPage({
                 { value: "2", label: "2000 level" },
                 { value: "3", label: "3000 level" },
                 { value: "4", label: "4000 level" },
+                { value: "2+", label: "2000 level or higher" },
+                { value: "3+", label: "3000 level or higher" },
               ],
             },
             {
@@ -142,14 +180,32 @@ export default async function CoursesPage({
                 { value: "Semester 2", label: "Semester 2" },
               ],
             },
+            ...(
+              [
+                ["college", "College", filterOptions.colleges],
+                ["area", "Area of interest", filterOptions.areas],
+                ["tag", "Tag", filterOptions.tags],
+              ] as const
+            ).flatMap(([key, label, values]) =>
+              // A filter with nothing to choose would only ever empty the list.
+              values.length
+                ? [
+                    {
+                      key,
+                      label,
+                      options: values.map((value) => ({ value, label: value })),
+                    },
+                  ]
+                : [],
+            ),
           ]}
         />
         <CourseDirectory
           courses={result.courses}
           page={result.page}
           pageSize={result.pageSize}
           total={result.total}
-          filtered={Boolean(query || level || session)}
+          filtered={Object.values(filters).some(Boolean)}
           academicYear={selectedAcademicYear}
           searchParams={paginationSearchParams}
         />

apps/web/lib/catalogue-import/kinds/course/adapter.ts

@@ -17,6 +17,7 @@ import {
   buildCourseExtractionUserPrompt,
 } from "./prompt.ts";
 import { fetchAnuCoursePage } from "./source.ts";
+import { loadKnownCourseTags } from "./tags.ts";
 
 export const courseKindAdapter: CatalogueSyncAdapter<CourseExtraction> = {
   kinds: ["course"],
@@ -45,10 +46,14 @@ export const courseKindAdapter: CatalogueSyncAdapter<CourseExtraction> = {
     });
   },
   buildSystemPrompt: buildCourseExtractionSystemPrompt,
-  buildUserPrompt(claim, pageMarkdown) {
+  async loadPromptContext(sql) {
+    return { knownTags: await loadKnownCourseTags(sql) };
+  },
+  buildUserPrompt(claim, pageMarkdown, context) {
     return buildCourseExtractionUserPrompt({
       expectedCode: claim.code,
       academicYear: claim.academicYear,
+      knownTags: context?.knownTags ?? [],
       pageMarkdown,
     });
   },

apps/web/lib/catalogue-import/kinds/course/contract.ts

@@ -153,6 +153,8 @@ export type CourseExtraction = {
   offeringStatus: "offered" | "not_offered" | "unknown";
   sourceUpdatedAt: string | null;
   areasOfInterest: string[];
+  /** Free-form categories degree rules can count units against. */
+  tags: string[];
   fees: CourseFee[];
   learningOutcomes: CourseLearningOutcome[];
   assessmentItems: CourseAssessmentItem[];
@@ -613,6 +615,7 @@ function validateExtractionShape(
       "offeringStatus",
       "sourceUpdatedAt",
       "areasOfInterest",
+      "tags",
       "fees",
       "learningOutcomes",
       "assessmentItems",
@@ -698,6 +701,9 @@ function validateExtractionShape(
     issues,
     (item, path) => requireString(item, path, issues),
   );
+  requireArray(record.tags, "$.tags", issues, (item, path) =>
+    requireString(item, path, issues),
+  );
   requireArray(record.fees, "$.fees", issues, (item, path) => {
     const fee = exactRecord(
       item,
@@ -1179,6 +1185,7 @@ export const COURSE_EXTRACTION_JSON_SCHEMA = {
     "offeringStatus",
     "sourceUpdatedAt",
     "areasOfInterest",
+    "tags",
     "fees",
     "learningOutcomes",
     "assessmentItems",
@@ -1255,6 +1262,7 @@ export const COURSE_EXTRACTION_JSON_SCHEMA = {
     offeringStatus: { enum: ["offered", "not_offered", "unknown"] },
     sourceUpdatedAt: { $ref: "#/$defs/nullableInstant" },
     areasOfInterest: { type: "array", items: { type: "string", minLength: 1 } },
+    tags: { type: "array", items: { type: "string", minLength: 1 } },
     fees: { type: "array", items: { $ref: "#/$defs/fee" } },
     learningOutcomes: { type: "array", items: { $ref: "#/$defs/outcome" } },
     assessmentItems: { type: "array", items: { $ref: "#/$defs/assessment" } },

apps/web/lib/catalogue-import/kinds/course/finalise.ts

@@ -63,6 +63,7 @@ export function emptyCourseExtraction({
     offeringStatus: "unknown",
     sourceUpdatedAt: null,
     areasOfInterest: [],
+    tags: [],
     fees: [],
     learningOutcomes: [],
     assessmentItems: [],

apps/web/lib/catalogue-import/kinds/course/project.ts

@@ -114,6 +114,7 @@ export type CourseSnapshotProjectionData = {
     sourceText: string;
   }>;
   areasOfInterest: Array<{ position: number; name: string }>;
+  tags: Array<{ position: number; name: string }>;
   attributes: Array<{
     position: number;
     attributeKind: CourseExtraction["attributes"][number]["attributeKind"];
@@ -857,6 +858,18 @@ export function projectCourseSnapshot(
     areasOfInterest.map(({ name }) => name),
     "areas of interest",
   );
+  // A tag is one category however it is capitalised, so a repeat collapses
+  // into the first spelling given.
+  const seenTags = new Set<string>();
+  const tags = (extraction.tags ?? [])
+    .map((name) => cleanText(name))
+    .filter((name) => {
+      const key = name.toLowerCase();
+      if (!name || seenTags.has(key)) return false;
+      seenTags.add(key);
+      return true;
+    })
+    .map((name, index) => ({ position: index + 1, name }));
   const attributes = rowsByPosition(extraction.attributes, "attributes").map(
     (attribute) => ({
       ...attribute,
@@ -984,6 +997,7 @@ export function projectCourseSnapshot(
     unitOptions: unitValue.options,
     fees,
     areasOfInterest,
+    tags,
     attributes,
     relatedCourses,
     courseOffering:

apps/web/lib/catalogue-import/kinds/course/prompt.ts

@@ -1,7 +1,7 @@
 import { COURSE_EXTRACTION_SCHEMA_VERSION } from "./contract.ts";
 
 export const COURSE_IMPORT_PARSER_VERSION = "coursemap-course-parser.v3";
-export const COURSE_IMPORT_PROMPT_VERSION = "coursemap-course-prompt.v4";
+export const COURSE_IMPORT_PROMPT_VERSION = "coursemap-course-prompt.v5";
 export const COURSE_SNAPSHOT_SCHEMA_VERSION = "course-snapshot.v1";
 
 /**
@@ -30,6 +30,11 @@ Source rules:
 10. classSummaryUrl is null or a complete HTTPS URL on programsandcourses.anu.edu.au taken from the page.
 11. Use null or [] when the page does not state something.
 
+Tags:
+- tags are short categories that degree rules count units against, such as "courses tagged as Science" or "from the Engineering list". Tag a course with every category the page supports: the discipline its college or school teaches (Science, Engineering, Business, Arts, Law, Medicine), and course types the page names, such as research project, capstone, internship or work-integrated learning.
+- When a known tag listed with the input fits, use it exactly as written. Coin a new tag only for a category no known tag covers, in title case and at most three words.
+- Give evidence for each tag under fieldKey tags, with confidence below 0.8 when the tag is inferred rather than stated.
+
 Writing the record:
 - Display text (introduction, description, workload, inherent requirements, prescribed texts, convener, delivery summary, assessment titles and learning outcomes) is copied from the page and tidied, never rewritten. Fix capitalisation, British English spelling, obvious typos and broken Markdown formatting, and drop page furniture such as "Back to the top". Do not summarise, shorten, reorder or add wording. Keep every course code, programme code, number, date, name and email address exactly as printed.
 - Every sourceText and evidence excerpt is the page's exact wording, untidied, so a reviewer can find it on the page.
@@ -53,11 +58,14 @@ Evidence and review:
 export function buildCourseExtractionUserPrompt({
   expectedCode,
   academicYear,
+  knownTags = [],
   pageMarkdown,
 }: {
   expectedCode: string;
   academicYear: number;
+  knownTags?: readonly string[];
   pageMarkdown: string;
 }) {
-  return `Expected course: ${expectedCode.toUpperCase()}\nSelected academic year: ${academicYear}\n\n${pageMarkdown}`;
+  const tags = knownTags.length ? `Known tags: ${knownTags.join("; ")}\n` : "";
+  return `Expected course: ${expectedCode.toUpperCase()}\nSelected academic year: ${academicYear}\n${tags}\n${pageMarkdown}`;
 }

apps/web/lib/catalogue-import/kinds/course/tags.ts

@@ -0,0 +1,33 @@
+import type { SyncSql } from "../../../catalogue-sync/sync-store.ts";
+
+const KNOWN_TAG_LIMIT = 200;
+
+/**
+ * The tag names already in use: those degree rules count units against, and
+ * those earlier courses were given. Offering them to the model keeps one
+ * category from splitting into near-duplicates such as Science and Sciences,
+ * which would stop a rule from matching the courses meant for it.
+ */
+export async function loadKnownCourseTags(sql: SyncSql): Promise<string[]> {
+  const rows = await sql`
+    select name
+    from (
+      select btrim(tag) as name from public.requirement_conditions
+      where tag is not null and btrim(tag) <> ''
+      union all
+      select name from public.course_tags
+    ) as used
+    group by lower(name), name
+    order by count(*) desc, name
+    limit ${KNOWN_TAG_LIMIT}
+  `;
+  const seen = new Set<string>();
+  return rows
+    .map((row) => String(row.name))
+    .filter((name) => {
+      const key = name.toLowerCase();
+      if (seen.has(key)) return false;
+      seen.add(key);
+      return true;
+    });
+}

apps/web/lib/catalogue-import/version-content.ts

@@ -174,6 +174,7 @@ async function readCourseContent(
     outcomes,
     assessments,
     links,
+    tags,
   ] = await Promise.all([
     sql`select position, units, label, source_text from public.course_unit_options where version_id = ${versionId} order by position`,
     sql`select * from public.course_fees where version_id = ${versionId} order by position`,
@@ -185,6 +186,7 @@ async function readCourseContent(
     sql`select id, position, body from public.course_learning_outcomes where version_id = ${versionId} order by position`,
     sql`select id, position, title, weight, hurdle, due_text, source_text from public.course_assessment_items where version_id = ${versionId} order by position`,
     sql`select assessment_item_id, learning_outcome_id from public.course_assessment_outcomes where version_id = ${versionId}`,
+    sql`select position, name from public.course_tags where version_id = ${versionId} order by position`,
   ]);
   const outcomePosition = new Map(
     outcomes.map((row) => [Number(row.id), Number(row.position)]),
@@ -242,6 +244,15 @@ async function readCourseContent(
       position: Number(row.position),
       name: String(row.name),
     })),
+    // Left out when empty, so versions from before tags hash as they did.
+    ...(tags.length
+      ? {
+          tags: tags.map((row) => ({
+            position: Number(row.position),
+            name: String(row.name),
+          })),
+        }
+      : {}),
     attributes: attributes.map((row) => ({
       position: Number(row.position),
       attributeKind: row.attribute_kind,

apps/web/lib/catalogue-sync/kind-adapter.ts

@@ -1,6 +1,12 @@
-import type { ClaimedCatalogueSync } from "./sync-store.ts";
+import type { ClaimedCatalogueSync, SyncSql } from "./sync-store.ts";
 import type { CatalogueKind, CatalogueContent } from "../catalogue/content.ts";
 
+/** Catalogue facts a prompt draws on beyond the page itself. */
+export type PromptContext = {
+  /** Tag names already in use, so the model reuses them rather than coining near-duplicates. */
+  knownTags: string[];
+};
+
 export type FetchedSourcePage = {
   sourceUrl: string;
   canonicalUrl: string;
@@ -50,7 +56,16 @@ export type CatalogueSyncAdapter<Extraction = unknown> = {
   /** The whole page as Markdown, which is also the model input. */
   prepareInput(claim: ClaimedCatalogueSync, page: FetchedSourcePage): string;
   buildSystemPrompt(): string;
-  buildUserPrompt(claim: ClaimedCatalogueSync, pageMarkdown: string): string;
+  /** Reads what the prompt needs from the catalogue, once per sync. */
+  loadPromptContext?(
+    sql: SyncSql,
+    claim: ClaimedCatalogueSync,
+  ): Promise<PromptContext>;
+  buildUserPrompt(
+    claim: ClaimedCatalogueSync,
+    pageMarkdown: string,
+    context?: PromptContext,
+  ): string;
   /** Strict validation of raw model output, recorded for the audit trail. */
   validateModelOutput(
     claim: ClaimedCatalogueSync,

apps/web/lib/catalogue-sync/persist-source-version.ts

@@ -134,6 +134,12 @@ async function insertCourseContent(
       values (${snapshotId}, ${area.position}, ${area.name})
     `;
   }
+  for (const tag of content.tags ?? []) {
+    await tx`
+      insert into public.course_tags (version_id, position, name)
+      values (${snapshotId}, ${tag.position}, ${tag.name})
+    `;
+  }
   for (const attribute of content.attributes) {
     await tx`
       insert into public.course_attributes (version_id, position, attribute_kind, value, source_text)