diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index cf075eaa..b87fd3ee 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -94,6 +94,7 @@ workflows: - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' '.github/workflows/self-test.yml': - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' + - 'denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed' '.github/workflows/signed-push-smoke.yml': - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' - 'actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1' diff --git a/.github/workflows/echidna-verify.yml b/.github/workflows/echidna-verify.yml index edca172f..57732cf8 100644 --- a/.github/workflows/echidna-verify.yml +++ b/.github/workflows/echidna-verify.yml @@ -18,12 +18,14 @@ on: paths: - 'lol/proofs/**' - 'a2ml/src/**/*.idr' + - 'a2ml/a2ml-core.ipkg' - 'avow-protocol/avow-lib/src/abi/*.idr' - '.github/workflows/echidna-verify.yml' pull_request: paths: - 'lol/proofs/**' - 'a2ml/src/**/*.idr' + - 'a2ml/a2ml-core.ipkg' - 'avow-protocol/avow-lib/src/abi/*.idr' schedule: # Weekly re-verification to catch stale-proof drift @@ -135,14 +137,16 @@ jobs: pack install-app idris2 idris2 --version - - name: Type-check A2ML core (Proofs + v1.1 profiles/base-vocab) + - name: Type-check normative A2ML core run: | + set -euo pipefail export PATH="$HOME/.pack/bin:$PATH" cd a2ml - idris2 --check src/A2ML/Proofs.idr 2>&1 | tee ../idris2-a2ml.log - # v1.1 additive core: checking Profiles.idr transitively type-checks - # BaseVocab.idr and TypedCore.idr (its imports). - idris2 --check src/A2ML/Profiles.idr 2>&1 | tee -a ../idris2-a2ml.log + # Use the package target: per-file `--check` can exit successfully + # when an imported module is missing and therefore is not a real + # gate for the authoritative module set (standards#556). + idris2 --version | tee ../idris2-a2ml.log + idris2 --typecheck a2ml-core.ipkg 2>&1 | tee -a ../idris2-a2ml.log - name: Upload log if: always() diff --git a/.github/workflows/self-test.yml b/.github/workflows/self-test.yml index 7df80804..29d3949c 100644 --- a/.github/workflows/self-test.yml +++ b/.github/workflows/self-test.yml @@ -35,6 +35,15 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # check-ts-allowlist-test.sh executes the generated Deno target, and the + # scorecard grounding suite runs pass-checks that use the same toolchain. + # Without installing Deno, the suite reported 18 assertion failures as + # one red test file and also made the scorecard fixtures fail. + - name: Install Deno test runtime + uses: denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed # v2.0.5 + with: + deno-version: v2.x + # PyYAML is required by the secret-scanner canary. The scorecard # grounding tests execute the same checks as registry-verify, including # checks that require ripgrep and xmllint. diff --git a/.machine_readable/REGISTRY.a2ml b/.machine_readable/REGISTRY.a2ml index a1b73d83..9976925f 100644 --- a/.machine_readable/REGISTRY.a2ml +++ b/.machine_readable/REGISTRY.a2ml @@ -45,7 +45,7 @@ name = "A2ML — Attested Markup Language" stream = "foundation" home = "a2ml/" canonical_doc = "a2ml/README.adoc" -source_hash = "sha256:480aeca64a094b3f184dc5aa98c00abb60ebcae33b41038410f2a5ff88d19427" +source_hash = "sha256:01e65e3c955746526e47b8a313070c13647228a1f6c453ad1b43d147789e664a" route = "the typed/verified machine-readable document format" [[spec]] diff --git a/a2ml/a2ml-core.ipkg b/a2ml/a2ml-core.ipkg index 407b9f12..03311279 100644 --- a/a2ml/a2ml-core.ipkg +++ b/a2ml/a2ml-core.ipkg @@ -16,12 +16,9 @@ authors = "Jonathan D.A. Jewell (hyperpolymath)" license = "MPL-2.0" sourcedir = "src" --- SCOPE. Every module listed here type-checks under idris2 0.7.0 and is --- gated in CI. A2ML.Converters is deliberately NOT listed: its renderers --- (toMarkdown/toDjot/toHtml/toLatex) are mutually recursive with their own --- where-block helpers, which cannot be total in that shape and needs a --- hand-done restructure. Tracked separately -- see the repo issue. Adding it --- here before that work is done would make this gate red on arrival. +-- SCOPE. Every normative core module listed here type-checks under Idris2 +-- 0.7.0 and is gated in CI. Converters is included now that its renderers use +-- structurally decreasing mutual recursion over List Block. modules = A2ML.TypedCore , A2ML.Surface , A2ML.Parser @@ -32,3 +29,4 @@ modules = A2ML.TypedCore , A2ML.CoreTests , A2ML.ParserTests , A2ML.Tests + , A2ML.Converters diff --git a/a2ml/src/A2ML/Converters.idr b/a2ml/src/A2ML/Converters.idr index 13a7a0e1..c718cdeb 100644 --- a/a2ml/src/A2ML/Converters.idr +++ b/a2ml/src/A2ML/Converters.idr @@ -1,8 +1,10 @@ module A2ML.Converters import A2ML.TypedCore +import A2ML.Parser import Data.String import Data.List +import Data.Maybe %default total @@ -10,36 +12,35 @@ import Data.List -- Markdown Converter -- ============================================================================ +mutual + markdownBlocks : List Block -> String + markdownBlocks [] = "" + markdownBlocks (block :: rest) = + markdownBlock block ++ markdownBlocks rest + + markdownBlock : Block -> String + markdownBlock (Section (MkSec sid title body)) = + let level = length sid.raw + heading = replicate level '#' ++ " " ++ title ++ "\n\n" + in heading ++ markdownBlocks body + markdownBlock (Para text) = text ++ "\n\n" + markdownBlock (Bullet items) = + concatMap (\item => "- " ++ item ++ "\n") items ++ "\n" + markdownBlock (Figure f) = + "![" ++ f.caption ++ "](#" ++ f.id.raw ++ ")\n\n" + markdownBlock (Table t) = + "**Table " ++ t.id.raw ++ ":** " ++ t.caption ++ "\n\n" + markdownBlock (Refs refs) = + "## References\n\n" ++ + concatMap (\r => "- " ++ r.label ++ "\n") refs ++ "\n" + markdownBlock (Opaque p) = + let lang = fromMaybe "" p.lang + in "```" ++ lang ++ "\n" ++ p.bytes ++ "\n```\n\n" + ||| Convert A2ML document to CommonMark Markdown export toMarkdown : Doc -> String -toMarkdown (MkDoc blocks) = concatMap blockToMd blocks - where - blockToMd : Block -> String - blockToMd (Section s) = - let level = length s.id.raw - heading = replicate level '#' ++ " " ++ s.title ++ "\n\n" - body = toMarkdown (MkDoc s.body) - in heading ++ body - - blockToMd (Para text) = text ++ "\n\n" - - blockToMd (Bullet items) = - concatMap (\item => "- " ++ item ++ "\n") items ++ "\n" - - blockToMd (Figure f) = - "![" ++ f.caption ++ "](#" ++ f.id.raw ++ ")\n\n" - - blockToMd (Table t) = - "**Table " ++ t.id.raw ++ ":** " ++ t.caption ++ "\n\n" - - blockToMd (Refs refs) = - "## References\n\n" ++ - concatMap (\r => "- " ++ r.label ++ "\n") refs ++ "\n" - - blockToMd (Opaque p) = - let lang = fromMaybe "" p.lang - in "```" ++ lang ++ "\n" ++ p.bytes ++ "\n```\n\n" +toMarkdown (MkDoc blocks) = markdownBlocks blocks -- ============================================================================ -- HTML Converter @@ -57,10 +58,56 @@ escapeHtml str = pack (concatMap escape (unpack str)) escape '\'' = unpack "'" escape c = [c] +mutual + htmlBlocks : List Block -> String + htmlBlocks [] = "" + htmlBlocks (block :: rest) = htmlBlock block ++ htmlBlocks rest + + htmlBlock : Block -> String + htmlBlock (Section (MkSec sid title body)) = + let level = min 6 (length sid.raw) -- HTML has h1-h6 + tag = "h" ++ show level + id_attr = " id=\"" ++ escapeHtml sid.raw ++ "\"" + in "<" ++ tag ++ id_attr ++ ">" ++ + escapeHtml title ++ + "\n" ++ + htmlBlocks body + htmlBlock (Para text) = + "

" ++ escapeHtml text ++ "

\n" + htmlBlock (Bullet items) = + "\n" + htmlBlock (Figure f) = + let id_attr = " id=\"" ++ escapeHtml f.id.raw ++ "\"" + in "\n" ++ + "
" ++ escapeHtml f.caption ++ "
\n" ++ + "\n" + htmlBlock (Table t) = + let id_attr = " id=\"" ++ escapeHtml t.id.raw ++ "\"" + in "\n" ++ + " " ++ escapeHtml t.caption ++ "\n" ++ + "\n" + htmlBlock (Refs refs) = + "
\n" ++ + "

References

\n" ++ + "
    \n" ++ + concatMap (\r => "
  1. " ++ escapeHtml r.label ++ "
  2. \n") refs ++ + "
\n" ++ + "
\n" + htmlBlock (Opaque p) = + let lang = fromMaybe "" p.lang + id_attr = case p.id of + Just id => " id=\"" ++ escapeHtml id.raw ++ "\"" + Nothing => "" + in "" ++ + escapeHtml p.bytes ++ + "\n" + ||| Convert A2ML document to HTML5 export toHtml : Doc -> String -toHtml doc = +toHtml (MkDoc blocks) = "\n" ++ "\n" ++ "\n" ++ @@ -75,99 +122,47 @@ toHtml doc = " \n" ++ "\n" ++ "\n" ++ - docToHtml doc ++ + htmlBlocks blocks ++ "\n" ++ "\n" - where - docToHtml : Doc -> String - docToHtml (MkDoc blocks) = concatMap blockToHtml blocks - - blockToHtml : Block -> String - blockToHtml (Section s) = - let level = min 6 (length s.id.raw) -- HTML has h1-h6 - tag = "h" ++ show level - id_attr = " id=\"" ++ escapeHtml s.id.raw ++ "\"" - in "<" ++ tag ++ id_attr ++ ">" ++ - escapeHtml s.title ++ - "\n" ++ - docToHtml (MkDoc s.body) - - blockToHtml (Para text) = - "

" ++ escapeHtml text ++ "

\n" - - blockToHtml (Bullet items) = - "
    \n" ++ - concatMap (\item => "
  • " ++ escapeHtml item ++ "
  • \n") items ++ - "
\n" - - blockToHtml (Figure f) = - let id_attr = " id=\"" ++ escapeHtml f.id.raw ++ "\"" - in "\n" ++ - "
" ++ escapeHtml f.caption ++ "
\n" ++ - "\n" - - blockToHtml (Table t) = - let id_attr = " id=\"" ++ escapeHtml t.id.raw ++ "\"" - in "\n" ++ - " " ++ escapeHtml t.caption ++ "\n" ++ - "\n" - - blockToHtml (Refs refs) = - "
\n" ++ - "

References

\n" ++ - "
    \n" ++ - concatMap (\r => "
  1. " ++ escapeHtml r.label ++ "
  2. \n") refs ++ - "
\n" ++ - "
\n" - - blockToHtml (Opaque p) = - let lang = fromMaybe "" p.lang - id_attr = case p.id of - Just id => " id=\"" ++ escapeHtml id.raw ++ "\"" - Nothing => "" - in "" ++ - escapeHtml p.bytes ++ - "\n" -- ============================================================================ -- Djot Converter -- ============================================================================ +mutual + djotBlocks : List Block -> String + djotBlocks [] = "" + djotBlocks (block :: rest) = djotBlock block ++ djotBlocks rest + + djotBlock : Block -> String + djotBlock (Section (MkSec sid title body)) = + let level = length sid.raw + heading = replicate level '#' ++ " " ++ title ++ "\n" ++ + "{#" ++ sid.raw ++ "}\n\n" + in heading ++ djotBlocks body + djotBlock (Para text) = text ++ "\n\n" + djotBlock (Bullet items) = + concatMap (\item => "- " ++ item ++ "\n") items ++ "\n" + djotBlock (Figure f) = + "!{#" ++ f.id.raw ++ "}[" ++ f.caption ++ "]\n\n" + djotBlock (Table t) = + "{#" ++ t.id.raw ++ "}\n" ++ + "**" ++ t.caption ++ "**\n\n" + djotBlock (Refs refs) = + "## References\n\n" ++ + concatMap (\r => ": " ++ r.label ++ "\n") refs ++ "\n" + djotBlock (Opaque p) = + let lang = fromMaybe "" p.lang + id_attr = case p.id of + Just id => "{#" ++ id.raw ++ "}\n" + Nothing => "" + in id_attr ++ "``` " ++ lang ++ "\n" ++ p.bytes ++ "\n```\n\n" + ||| Convert A2ML document to Djot markup export toDjot : Doc -> String -toDjot (MkDoc blocks) = concatMap blockToDjot blocks - where - blockToDjot : Block -> String - blockToDjot (Section s) = - let level = length s.id.raw - heading = replicate level '#' ++ " " ++ s.title ++ "\n" ++ - "{#" ++ s.id.raw ++ "}\n\n" - body = toDjot (MkDoc s.body) - in heading ++ body - - blockToDjot (Para text) = text ++ "\n\n" - - blockToDjot (Bullet items) = - concatMap (\item => "- " ++ item ++ "\n") items ++ "\n" - - blockToDjot (Figure f) = - "!{#" ++ f.id.raw ++ "}[" ++ f.caption ++ "]\n\n" - - blockToDjot (Table t) = - "{#" ++ t.id.raw ++ "}\n" ++ - "**" ++ t.caption ++ "**\n\n" - - blockToDjot (Refs refs) = - "## References\n\n" ++ - concatMap (\r => ": " ++ r.label ++ "\n") refs ++ "\n" - - blockToDjot (Opaque p) = - let lang = fromMaybe "" p.lang - id_attr = case p.id of - Just id => "{#" ++ id.raw ++ "}\n" - Nothing => "" - in id_attr ++ "``` " ++ lang ++ "\n" ++ p.bytes ++ "\n```\n\n" +toDjot (MkDoc blocks) = djotBlocks blocks -- ============================================================================ -- LaTeX Converter @@ -190,10 +185,57 @@ escapeTex str = pack (concatMap escape (unpack str)) escape '~' = unpack "\\~{}" escape c = [c] +mutual + latexBlocks : List Block -> String + latexBlocks [] = "" + latexBlocks (block :: rest) = latexBlock block ++ latexBlocks rest + + latexBlock : Block -> String + latexBlock (Section (MkSec sid title body)) = + let level = length sid.raw + command = case level of + 1 => "\\section" + 2 => "\\subsection" + 3 => "\\subsubsection" + 4 => "\\paragraph" + _ => "\\subparagraph" + label = "\\label{" ++ sid.raw ++ "}" + in command ++ "{" ++ escapeTex title ++ "}" ++ label ++ "\n" ++ + latexBlocks body ++ "\n" + latexBlock (Para text) = escapeTex text ++ "\n\n" + latexBlock (Bullet items) = + "\\begin{itemize}\n" ++ + concatMap (\item => " \\item " ++ escapeTex item ++ "\n") items ++ + "\\end{itemize}\n\n" + latexBlock (Figure f) = + "\\begin{figure}[h]\n" ++ + " \\centering\n" ++ + " % Insert figure here\n" ++ + " \\caption{" ++ escapeTex f.caption ++ "}\n" ++ + " \\label{fig:" ++ f.id.raw ++ "}\n" ++ + "\\end{figure}\n\n" + latexBlock (Table t) = + "\\begin{table}[h]\n" ++ + " \\centering\n" ++ + " \\caption{" ++ escapeTex t.caption ++ "}\n" ++ + " \\label{tab:" ++ t.id.raw ++ "}\n" ++ + " % Insert table here\n" ++ + "\\end{table}\n\n" + latexBlock (Refs refs) = + "\\section*{References}\n" ++ + "\\begin{enumerate}\n" ++ + concatMap (\r => " \\item " ++ escapeTex r.label ++ "\n") refs ++ + "\\end{enumerate}\n\n" + latexBlock (Opaque p) = + let lang = fromMaybe "" p.lang + in "\\begin{lstlisting}[language=" ++ lang ++ "]\n" ++ + p.bytes ++ "\n" ++ + "\\end{lstlisting}\n\n" + ||| Convert A2ML document to LaTeX export toLatex : Doc -> String -toLatex doc = +toLatex (MkDoc blocks) = "\\documentclass{article}\n" ++ "\\usepackage[utf8]{inputenc}\n" ++ "\\usepackage{hyperref}\n" ++ @@ -201,98 +243,47 @@ toLatex doc = "\n" ++ "\\begin{document}\n" ++ "\n" ++ - docToTex doc ++ + latexBlocks blocks ++ "\n" ++ "\\end{document}\n" - where - docToTex : Doc -> String - docToTex (MkDoc blocks) = concatMap blockToTex blocks - - blockToTex : Block -> String - blockToTex (Section s) = - let level = length s.id.raw - command = case level of - 1 => "\\section" - 2 => "\\subsection" - 3 => "\\subsubsection" - 4 => "\\paragraph" - _ => "\\subparagraph" - label = "\\label{" ++ s.id.raw ++ "}" - in command ++ "{" ++ escapeTex s.title ++ "}" ++ label ++ "\n" ++ - docToTex (MkDoc s.body) ++ "\n" - - blockToTex (Para text) = - escapeTex text ++ "\n\n" - - blockToTex (Bullet items) = - "\\begin{itemize}\n" ++ - concatMap (\item => " \\item " ++ escapeTex item ++ "\n") items ++ - "\\end{itemize}\n\n" - - blockToTex (Figure f) = - "\\begin{figure}[h]\n" ++ - " \\centering\n" ++ - " % Insert figure here\n" ++ - " \\caption{" ++ escapeTex f.caption ++ "}\n" ++ - " \\label{fig:" ++ f.id.raw ++ "}\n" ++ - "\\end{figure}\n\n" - - blockToTex (Table t) = - "\\begin{table}[h]\n" ++ - " \\centering\n" ++ - " \\caption{" ++ escapeTex t.caption ++ "}\n" ++ - " \\label{tab:" ++ t.id.raw ++ "}\n" ++ - " % Insert table here\n" ++ - "\\end{table}\n\n" - - blockToTex (Refs refs) = - "\\section*{References}\n" ++ - "\\begin{enumerate}\n" ++ - concatMap (\r => " \\item " ++ escapeTex r.label ++ "\n") refs ++ - "\\end{enumerate}\n\n" - - blockToTex (Opaque p) = - let lang = fromMaybe "" p.lang - in "\\begin{lstlisting}[language=" ++ lang ++ "]\n" ++ - p.bytes ++ "\n" ++ - "\\end{lstlisting}\n\n" -- ============================================================================ -- Plain Text Converter -- ============================================================================ +numberedRefs : Nat -> List Ref -> String +numberedRefs _ [] = "" +numberedRefs n (ref :: rest) = + show n ++ ". " ++ ref.label ++ "\n" ++ numberedRefs (S n) rest + +mutual + plainTextBlocks : List Block -> String + plainTextBlocks [] = "" + plainTextBlocks (block :: rest) = + plainTextBlock block ++ plainTextBlocks rest + + plainTextBlock : Block -> String + plainTextBlock (Section (MkSec _ title body)) = + title ++ "\n" ++ + replicate (length title) '=' ++ "\n\n" ++ + plainTextBlocks body + plainTextBlock (Para text) = text ++ "\n\n" + plainTextBlock (Bullet items) = + concatMap (\item => "* " ++ item ++ "\n") items ++ "\n" + plainTextBlock (Figure f) = + "[Figure: " ++ f.caption ++ "]\n\n" + plainTextBlock (Table t) = + "[Table: " ++ t.caption ++ "]\n\n" + plainTextBlock (Refs refs) = + "References\n" ++ + "==========\n\n" ++ + numberedRefs 1 refs ++ "\n" + plainTextBlock (Opaque p) = p.bytes ++ "\n\n" + ||| Convert A2ML document to plain text (no formatting) export toPlainText : Doc -> String -toPlainText (MkDoc blocks) = concatMap blockToText blocks - where - blockToText : Block -> String - blockToText (Section s) = - s.title ++ "\n" ++ - replicate (length s.title) '=' ++ "\n\n" ++ - toPlainText (MkDoc s.body) - - blockToText (Para text) = text ++ "\n\n" - - blockToText (Bullet items) = - concatMap (\item => "* " ++ item ++ "\n") items ++ "\n" - - blockToText (Figure f) = - "[Figure: " ++ f.caption ++ "]\n\n" - - blockToText (Table t) = - "[Table: " ++ t.caption ++ "]\n\n" - - blockToText (Refs refs) = - "References\n" ++ - "==========\n\n" ++ - concatMap (\r => show (1 + elemIndex r refs) ++ ". " ++ r.label ++ "\n") refs ++ "\n" - where - elemIndex : Eq a => a -> List a -> Nat - elemIndex x [] = 0 - elemIndex x (y :: ys) = if x == y then 0 else 1 + elemIndex x ys - - blockToText (Opaque p) = p.bytes ++ "\n\n" +toPlainText (MkDoc blocks) = plainTextBlocks blocks -- ============================================================================ -- Converter Selection @@ -344,6 +335,23 @@ testMarkdownRoundTrip doc = Success doc' _ => length (blocks doc) == length (blocks doc') _ => False +mutual + extractContent : Doc -> String + extractContent (MkDoc blocks) = extractBlocks blocks + + extractBlocks : List Block -> String + extractBlocks [] = "" + extractBlocks (block :: rest) = extractBlock block ++ extractBlocks rest + + extractBlock : Block -> String + extractBlock (Section (MkSec _ title body)) = title ++ extractBlocks body + extractBlock (Para text) = text + extractBlock (Bullet items) = concatMap id items + extractBlock (Figure f) = f.caption + extractBlock (Table t) = t.caption + extractBlock (Refs refs) = concatMap (\r => r.label) refs + extractBlock (Opaque p) = p.bytes + ||| Test if A2ML → Plain Text → A2ML preserves content export partial @@ -353,15 +361,3 @@ testPlainTextRoundTrip doc = content1 = extractContent doc content2 = txt in length content1 == length content2 - where - extractContent : Doc -> String - extractContent (MkDoc blocks) = concatMap extractBlock blocks - - extractBlock : Block -> String - extractBlock (Section s) = s.title ++ extractContent (MkDoc s.body) - extractBlock (Para text) = text - extractBlock (Bullet items) = concatMap id items - extractBlock (Figure f) = f.caption - extractBlock (Table t) = t.caption - extractBlock (Refs refs) = concatMap (\r => r.label) refs - extractBlock (Opaque p) = p.bytes