Adopt the relix-core#102 engine changes and drop their workarounds

docs/design.md

@@ -327,7 +327,7 @@ Everything goes through `com.darkcollective.relix.embed`:
 | queries | `script(text)`, `relation(expression)` |
 | rows | `Relation.stream()` (closed on `EPIPE`/`SIGINT`) |
 | `explain`, `optimize`, `trace`, `bundle`, `ir`, `provenance` | `explain`/`explainJson`, `optimized().render()` + `rewrites()`, `stream(listener)`, `renderJson`, `model().ir()`, `provenance(semiring[, weight])` |
-| `fmt` | the statement printer behind `Relix.definitions()` |
+| `fmt` | `ScriptPrinter.print(Script, Spelling)`, which keeps comments |
 | `catalog ls/schema` | `relation("relix.relations")` etc. |
 | `doc` | the reference lookup in `relix-docs` (R5): language pages and the installed functions' pages, one index |
 
@@ -460,7 +460,8 @@ keyword or name, across the language pages and the pages of the installed functi
 a language keyword keeping a shared name (`fix` is the operator; `Fix` the function is
 found as a function). Every program that shows the reference needs that same index, so it
 belongs beside the pages it indexes rather than in each front end. Filed as
-DarkCollective/relix-core#95.
+DarkCollective/relix-core#95, and done: `relix doc` looks pages up in relix-docs'
+`ReferenceLookup`.
 
 ## 10. Decisions
 

src/main/java/com/darkcollective/relix/cli/command/DocCommand.java

@@ -19,16 +19,20 @@
 import com.darkcollective.relix.cli.ExitCode;
 import com.darkcollective.relix.cli.io.Pager;
 import com.darkcollective.relix.cli.io.RowSink;
-import com.darkcollective.relix.cli.render.DocIndex;
 import com.darkcollective.relix.cli.render.DocRenderer;
+import com.darkcollective.relix.docs.ReferenceLookup;
 import com.darkcollective.relix.docs.ReferencePage;
+import com.darkcollective.relix.docs.RelixDocs;
+import com.darkcollective.relix.function.FunctionCatalog;
 import com.darkcollective.relix.symbol.ScalarType;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Mixin;
 import picocli.CommandLine.Option;
 import picocli.CommandLine.Parameters;
 import picocli.CommandLine.ParentCommand;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.Callable;
 
@@ -40,6 +44,9 @@
  * function, so {@code relix doc fix} is the recursion operator and
  * {@code relix doc --function fix} is the function. The page is rendered for a terminal
  * and, on one, shown through {@code $PAGER}. With no topic, the pages are listed, as rows.
+ *
+ * <p>Pages are found by {@code relix-docs}' {@link ReferenceLookup}, over the language pages
+ * and the installed functions' pages (DarkCollective/relix-core#95).
  */
 @Command(name = "doc",
         mixinStandardHelpOptions = true,
@@ -75,16 +82,16 @@ final class DocCommand implements Callable<Integer> {
     @Override
     public Integer call() {
         Invocation invocation = root.invocation(false);
-        DocIndex index = DocIndex.installed();
+        ReferenceLookup index = ReferenceLookup.of(FunctionCatalog.discover());
         if (topic == null) {
             list(invocation, index);
             return ExitCode.SUCCESS.status();
         }
-        Optional<ReferencePage> found = function ? index.function(topic) : index.lookup(topic);
+        Optional<ReferencePage> found = function ? index.function(topic) : index.entry(topic);
         ReferencePage page = found.orElseThrow(() -> new CommandFailure(ExitCode.USAGE,
                 "no reference page for '" + topic + "'" + (function ? " among the functions" : "")
                         + "; relix doc" + (function ? " --function" : "") + " lists them"));
-        String text = index.markdown(page).orElseThrow(() -> new CommandFailure(ExitCode.INTERNAL,
+        String text = index.page(page).orElseThrow(() -> new CommandFailure(ExitCode.INTERNAL,
                 "the reference lists " + page.path() + " but has no page there"));
         RowSink out = new RowSink(invocation.host().out(), false);
         if (!markdown) {
@@ -98,13 +105,17 @@ public Integer call() {
         return ExitCode.SUCCESS.status();
     }
 
-    private void list(Invocation invocation, DocIndex index) {
+    /** The language pages, in the reference's order, unless --function; then the functions'. */
+    private void list(Invocation invocation, ReferenceLookup index) {
         Listing listing = new Listing("topic", ScalarType.STRING, "category", ScalarType.STRING,
                 "title", ScalarType.STRING, "summary", ScalarType.STRING);
-        for (ReferencePage page : index.pages()) {
-            if (!function || page.category().equals(DocIndex.FUNCTION)) {
-                listing.add(page.symbol(), page.category(), page.title(), page.summary());
-            }
+        List<ReferencePage> pages = new ArrayList<>();
+        if (!function) {
+            pages.addAll(RelixDocs.referencePages());
+        }
+        index.functionsByCategory().values().forEach(pages::addAll);
+        for (ReferencePage page : pages) {
+            listing.add(page.symbol(), page.category(), page.title(), page.summary());
         }
         listing.print(invocation, invocation.format(formatting), "reference");
     }

src/main/java/com/darkcollective/relix/cli/command/FmtCommand.java

@@ -15,6 +15,7 @@
  */
 package com.darkcollective.relix.cli.command;
 
+import com.darkcollective.relix.ast.Spelling;
 import com.darkcollective.relix.cli.CommandFailure;
 import com.darkcollective.relix.cli.ExitCode;
 import com.darkcollective.relix.cli.io.Reporter;
@@ -41,7 +42,7 @@
 /**
  * {@code relix fmt}: prints scripts canonically (design §3), as {@code gofmt} does Go.
  *
- * <p>Each script is printed by the engine's statement printer, with its comments kept
+ * <p>Each script is printed by the engine's script printer, with its comments kept
  * (see {@link ScriptFormatter}). With no option the formatted scripts go to standard
  * output; {@code -w} rewrites each file that changes in place, and {@code --check} changes
  * nothing, names each file that would change, and exits 1 if any would, for CI.
@@ -97,8 +98,7 @@ public Integer call() {
             throw new CommandFailure(ExitCode.USAGE, "-w and --check cannot be given together");
         }
         Invocation invocation = root.invocation(false);
-        ScriptFormatter.Spelling chosen = spelling.keywords
-                ? ScriptFormatter.Spelling.KEYWORDS : ScriptFormatter.Spelling.GLYPHS;
+        Spelling chosen = spelling.keywords ? Spelling.KEYWORDS : Spelling.GLYPHS;
         List<ScriptSource> sources = sources(invocation);
         if (write && sources.stream().anyMatch(s -> s.name().equals(ScriptSource.STDIN))) {
             throw new CommandFailure(ExitCode.USAGE, "-w rewrites files; standard input has none to rewrite");
@@ -110,7 +110,7 @@ public Integer call() {
         for (ScriptSource source : sources) {
             String formatted;
             try {
-                formatted = ScriptFormatter.format(source.text(), chosen);
+                formatted = ScriptFormatter.format(source.text(), source.name(), chosen);
             } catch (ScriptParseException e) {
                 reporter.error(new Reporter.Place(source.name(), e.line(), e.column()) + ": error: "
                         + e.description());

src/main/java/com/darkcollective/relix/cli/command/OptimizeCommand.java

@@ -17,14 +17,11 @@
 
 import com.darkcollective.relix.cli.CommandFailure;
 import com.darkcollective.relix.cli.ExitCode;
-import com.darkcollective.relix.cli.io.Reporter;
 import com.darkcollective.relix.cli.io.RowSink;
 import com.darkcollective.relix.cli.io.ScriptText;
 import com.darkcollective.relix.cli.report.OptimizationReport;
 import com.darkcollective.relix.embed.Relation;
-import com.darkcollective.relix.embed.Relix;
 import com.darkcollective.relix.lang.ast.QueryStatement;
-import com.darkcollective.relix.lang.ast.ScriptParseException;
 import picocli.CommandLine.Command;
 import picocli.CommandLine.Option;
 
@@ -38,9 +35,9 @@
  *
  * <p>Only the {@code query} statements change, each to {@code query { … };} around its
  * rewritten expression, in which the views it reads are inlined. Every other statement,
- * and every comment, is left as written. A rewrite the engine cannot write back as text
- * that parses is not printed: that query is left as written too, with a warning, so the
- * output is always a script that runs. With {@code --report}, what is printed instead is
+ * and every comment, is left as written. The engine writes every rewrite back as Relix
+ * that parses (DarkCollective/relix-core#100), so the output is a script that runs. With
+ * {@code --report}, what is printed instead is
  * which rules fired, and each query before and after.
  */
 @Command(name = "optimize",
@@ -65,7 +62,7 @@ ScriptRunner.View view(Invocation invocation) {
                 out.text(OptimizationReport.generate(queries.isEmpty()
                         ? script.session().model() : queries.getFirst().model(), entries(queries)));
             } else {
-                out.text(rewritten(script, queries, invocation.reporter()));
+                out.text(rewritten(script, queries));
             }
             return ExitCode.SUCCESS;
         };
@@ -83,7 +80,7 @@ private static List<OptimizationReport.Entry> entries(List<Relation> queries) {
     }
 
     /** The text with each query statement replaced by its rewritten query, in order. */
-    private static String rewritten(ScriptRunner.Script script, List<Relation> queries, Reporter reporter) {
+    private static String rewritten(ScriptRunner.Script script, List<Relation> queries) {
         String text = script.source().text();
         List<ScriptText.Piece> statements = ScriptText.of(text).statements().stream()
                 .filter(piece -> piece.statement() instanceof QueryStatement)
@@ -96,28 +93,12 @@ private static String rewritten(ScriptRunner.Script script, List<Relation> queri
         int at = 0;
         for (int i = 0; i < statements.size(); i++) {
             ScriptText.Span span = statements.get(i).span();
-            Relation query = queries.get(i);
-            String rewritten = "query { " + query.optimized().render() + " };";
             out.append(text, at, span.start());
-            if (parses(rewritten)) {
-                out.append(rewritten);
-            } else {
-                reporter.warning(script.source().name() + ": " + query.label().orElse("a query")
-                        + ": the engine cannot write its rewrite as Relix that parses; left as written");
-                out.append(span.of(text));
-            }
+            out.append("query { ").append(queries.get(i).optimized().render()).append(" };");
             at = span.end();
         }
         out.append(text, at, text.length());
         return out.toString();
     }
 
-    private static boolean parses(String statement) {
-        try {
-            Relix.parse(statement);
-            return true;
-        } catch (ScriptParseException e) {
-            return false;
-        }
-    }
 }

src/main/java/com/darkcollective/relix/cli/io/ScriptText.java

@@ -25,12 +25,11 @@
 import java.util.Objects;
 
 /**
- * A script's text together with what it parses to: where in the text each statement is,
- * and where its comments are.
+ * A script's text together with what it parses to: where in the text each statement is.
  *
  * <p>The parser gives each statement the place it starts; this finds where it ends, at the
- * semicolon that closes it, and finds the comments. Those are the language's, read as its
- * two lexers read them: {@code --} to the end of the line and {@code /* *}{@code /}, outside
+ * semicolon that closes it, passing over the comments a semicolon may be in. Those are the
+ * language's, read as its two lexers read them: {@code --} to the end of the line and {@code /* *}{@code /}, outside
  * a string. Within an inline table only a line that begins with one is a comment, since a
  * cell may hold {@code --} as data, and a table's {@code |---|} rule is not one.
  */
@@ -60,25 +59,21 @@ public String of(String text) {
      *
      * @param statement what it parses to
      * @param span      its text, from its first character to the semicolon that ends it
-     * @param comments  the comments within that text
      */
-    public record Piece(Statement statement, Span span, List<Span> comments) {
+    public record Piece(Statement statement, Span span) {
 
-        /** Checks every component and copies the comments. */
+        /** Checks every component. */
         public Piece {
             Objects.requireNonNull(statement, "statement");
             Objects.requireNonNull(span, "span");
-            comments = List.copyOf(comments);
         }
     }
 
     private final String text;
-    private final Script script;
     private final List<Piece> pieces;
 
-    private ScriptText(String text, Script script, List<Piece> pieces) {
+    private ScriptText(String text, List<Piece> pieces) {
         this.text = text;
-        this.script = script;
         this.pieces = List.copyOf(pieces);
     }
 
@@ -97,40 +92,21 @@ public static ScriptText of(String text) {
             int start = offset(statement.location(), lines, text);
             Scan scan = new Scan(text, start);
             scan.toEndOfStatement();
-            pieces.add(new Piece(statement, new Span(start, scan.position), scan.comments));
+            pieces.add(new Piece(statement, new Span(start, scan.position)));
         }
-        return new ScriptText(text, script, pieces);
+        return new ScriptText(text, pieces);
     }
 
     /** {@return the text} */
     public String text() {
         return text;
     }
 
-    /** {@return what the text parses to} */
-    public Script script() {
-        return script;
-    }
-
     /** {@return each statement and where it is written, in order} */
     public List<Piece> statements() {
         return pieces;
     }
 
-    /**
-     * The comments in a stretch of the text that holds nothing else but white space, such as
-     * the stretch between two statements.
-     *
-     * @param from where the stretch starts
-     * @param to   where it ends
-     * @return its comments, in order
-     */
-    public List<Span> comments(int from, int to) {
-        Scan scan = new Scan(text.substring(0, to), from);
-        scan.toEnd();
-        return scan.comments;
-    }
-
     private static int[] lineStarts(String text) {
         List<Integer> starts = new ArrayList<>();
         starts.add(0);
@@ -148,17 +124,16 @@ private static int offset(SourceLocation location, int[] lines, String text) {
     }
 
     /**
-     * Reads the text as the lexers do, as far as finding comments and the end of a
-     * statement needs: through strings, delimited names, braces (an expression body) and
-     * brackets (an inline table).
+     * Reads the text as the lexers do, as far as finding the end of a statement needs:
+     * through comments, strings, delimited names, braces (an expression body) and brackets
+     * (an inline table).
      */
     private static final class Scan {
 
         private enum Context { TOP, BRACE, BRACKET }
 
         private final String text;
         private final List<Context> stack = new ArrayList<>(List.of(Context.TOP));
-        private final List<Span> comments = new ArrayList<>();
         private int position;
 
         Scan(String text, int start) {
@@ -177,13 +152,6 @@ void toEndOfStatement() {
             }
         }
 
-        /** Reads to the end of the text. */
-        void toEnd() {
-            while (position < text.length()) {
-                step();
-            }
-        }
-
         private void step() {
             char c = text.charAt(position);
             Context context = stack.getLast();
@@ -236,23 +204,21 @@ private void step() {
             }
         }
 
-        /** Reads a comment starting here, if one does. */
+        /** Reads past a comment starting here, if one does. */
         private boolean comment(Context context) {
             if (!text.startsWith("--", position) && !text.startsWith("/*", position)) {
                 return false;
             }
             if (context == Context.BRACKET && !atLineStart()) {
                 return false;
             }
-            int start = position;
             if (text.startsWith("--", position)) {
                 int end = text.indexOf('\n', position);
                 position = end < 0 ? text.length() : end;
             } else {
                 int end = text.indexOf("*/", position + 2);
                 position = end < 0 ? text.length() : end + 2;
             }
-            comments.add(new Span(start, position));
             return true;
         }