diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index dcf953e..1779d9a 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -672,7 +672,7 @@ jobs: env: KEY: ${{ matrix.key }} PY: ${{ matrix.python }} - EXAMPLES: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py" + EXAMPLES: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py 3[0-9]_*.py" EXAMPLES_SKIP_FILE: scripts/native_examples_skip.txt # The Hugging Face cache inside the workspace, as in test-python-examples.yml: one path on every platform, cacheable. HF_HOME: ${{ github.workspace }}/.hf-cache diff --git a/.github/workflows/test-python-examples.yml b/.github/workflows/test-python-examples.yml index 4952083..ed797f6 100644 --- a/.github/workflows/test-python-examples.yml +++ b/.github/workflows/test-python-examples.yml @@ -60,7 +60,7 @@ on: description: "Glob pattern(s) for examples to run (space-separated, relative to examples)." required: false type: string - default: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py" + default: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py 3[0-9]_*.py" build-version: description: "Override package version (PEP 440) for build.sh" required: false @@ -76,7 +76,7 @@ on: examples: description: "Glob pattern(s) for examples to run (space-separated, relative to examples)." required: false - default: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py" + default: "0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py 3[0-9]_*.py" # SUPERSEDE, do not race (same rule in test-jvm.yml). # @@ -113,7 +113,7 @@ env: # The image download-jars takes the jars from. Unset (push, pull_request, schedule) # means latest; a release build passes its tag. ARCADEDB_IMAGE_TAG: ${{ inputs.arcadedb-tag || 'latest' }} - EXAMPLES: ${{ inputs.examples || '0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py' }} + EXAMPLES: ${{ inputs.examples || '0[1-9]_*.py 1[0-9]_*.py 2[0-9]_*.py 3[0-9]_*.py' }} # Note: Example 21 runs in CI at reduced scale; its arguments and 15-minute # limit are set in the per-example case in scripts/run_examples.sh, which the diff --git a/README.md b/README.md index b6df9c5..54a09a9 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ This prints `Name: Alice` (the engine also logs to standard error). `create_data - Graph, document, key/value, vector (JVector HNSW) and time series models; ACID transactions. - Import from CSV, XML and ArcadeDB JSONL; export to JSONL. - Columnar results with `to_columns()`, `to_arrow()` and `to_dataframe()`. +- Faceted queries: the `FACETS` clause of a `SELECT` with a `COLUMNAR` index, and the counts as a dict in `ResultSet.facets` (needs engine 26.12.1 or later). - An optional in-process HTTP server (`create_server()`), off until you call it, for other processes that need the same data. @@ -82,13 +83,13 @@ Releases from this repository start with 26.11.1. The 26.10.x wheels were built ## Documentation -- Here: [`docs/`](https://github.com/ArcadeData/arcadedb-python/tree/main/docs/modules/ROOT/pages) is an Antora component (AsciiDoc) with Installation, [Which package?](https://github.com/ArcadeData/arcadedb-python/blob/main/docs/modules/ROOT/pages/which-package.adoc), Quickstart, Databases and queries, Vectors, and Concurrency, processes and crashes. It covers only what is specific to the Python bindings. +- Here: [`docs/`](https://github.com/ArcadeData/arcadedb-python/tree/main/docs/modules/ROOT/pages) is an Antora component (AsciiDoc) with Installation, [Which package?](https://github.com/ArcadeData/arcadedb-python/blob/main/docs/modules/ROOT/pages/which-package.adoc), Quickstart, Databases and queries, Vectors, Faceted queries, and Concurrency, processes and crashes. It covers only what is specific to the Python bindings. - Published at [docs.arcadedb.com/arcadedb-python](https://docs.arcadedb.com/arcadedb-python/), the same site as the main ArcadeDB manual, rebuilt daily from `main` (ArcadeData/arcadedb-docs#505). The `Docs` workflow builds it as a site of its own on every change under `docs/` and attaches it as the `docs-site` artifact. To build it locally, run `npx -p @antora/cli@3.2.1 -p @antora/site-generator@3.2.1 antora docs/antora-playbook.yml` and open `docs/build/site/index.html`. - ArcadeDB itself (SQL, openCypher, schema, server): [docs.arcadedb.com](https://docs.arcadedb.com). It is not repeated here. ## Examples -[`examples/`](https://github.com/ArcadeData/arcadedb-python/tree/main/examples) holds 29 runnable scripts, from a first document store to vector search, time series, graph algorithms and server mode. CI runs them ([`test-python-examples.yml`](https://github.com/ArcadeData/arcadedb-python/blob/main/.github/workflows/test-python-examples.yml)). +[`examples/`](https://github.com/ArcadeData/arcadedb-python/tree/main/examples) holds 30 runnable scripts, from a first document store to vector search, time series, graph algorithms and server mode. CI runs them ([`test-python-examples.yml`](https://github.com/ArcadeData/arcadedb-python/blob/main/.github/workflows/test-python-examples.yml)). ## Build and test from source diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 48d1813..c0d9de6 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -16,6 +16,7 @@ ** xref:how-to/functions-and-events.adoc[Python functions and record events] ** xref:how-to/vectors.adoc[Vectors] ** xref:how-to/columnar-index.adoc[The COLUMNAR index and its settings] +** xref:how-to/facets.adoc[Faceted queries] * Development ** xref:development/ci.adoc[Continuous integration] ** xref:development/release.adoc[Release] diff --git a/docs/modules/ROOT/pages/how-to/columnar-index.adoc b/docs/modules/ROOT/pages/how-to/columnar-index.adoc index 68c77b6..9acd40b 100644 --- a/docs/modules/ROOT/pages/how-to/columnar-index.adoc +++ b/docs/modules/ROOT/pages/how-to/columnar-index.adoc @@ -3,7 +3,7 @@ // Sources: ArcadeDB 26.12.1: GlobalConfiguration.java (COLUMNAR_*), AlterDatabaseStatement.java, ColumnarGroupByStep.java, ColumnarCrashRecoveryTest.java. The statements about Python are pinned by tests/test_columnar_index.py, which runs on arcadedb-jvm and on the native arcadedb. -A `COLUMNAR` index keeps a column projection of a type next to its records. The engine applies, flushes and merges it in threads of its own, and the `FACETS` clause and `GROUP BY` read it. It needs engine 26.12.1 or later: an older engine refuses `CREATE INDEX ... COLUMNAR` with `Index type 'COLUMNAR' is not supported`. The facets themselves are read with `ResultSet.facets`. +A `COLUMNAR` index keeps a column projection of a type next to its records. The engine applies, flushes and merges it in threads of its own, and the `FACETS` clause and `GROUP BY` read it. It needs engine 26.12.1 or later: an older engine refuses `CREATE INDEX ... COLUMNAR` with `Index type 'COLUMNAR' is not supported`. The facets themselves are read with `ResultSet.facets`, which xref:how-to/facets.adoc[Faceted queries] shows kind by kind. The index is a statement, so it is the same on both builds: diff --git a/docs/modules/ROOT/pages/how-to/facets.adoc b/docs/modules/ROOT/pages/how-to/facets.adoc new file mode 100644 index 0000000..e14bacb --- /dev/null +++ b/docs/modules/ROOT/pages/how-to/facets.adoc @@ -0,0 +1,662 @@ += Faceted queries +:description: Count the values of a property next to the rows of a SQL query with the FACETS clause, and read the counts from Python as the dict ResultSet.facets, on both packages. + +// Sources: ArcadeDB 26.12.1: the FACETS clause, the COLUMNAR index and facet.searchValues (query/facet). The statements about Python are pinned by tests/test_result_set_facets.py, tests/test_result_set_facets_kinds.py and tests/test_columnar_index.py, which run on arcadedb-jvm and on the native arcadedb. Every block on this page is run by tests/test_docs_facets_page.py, and examples/30_columnar_facets.py is the same on a larger data set. + +A facet is the count of the records for each value of a property, next to the rows of the query. A search page shows them as the numbers in a sidebar: 5 records in category `a`, 2 in category `b`. The `FACETS` clause of a `SELECT` asks for them, and `ResultSet.facets` returns them as a plain dict. They cover all the records the `WHERE` clause matched, whatever `SKIP` and `LIMIT` kept for the rows, and a terms facet gives the same counts as a `GROUP BY` on the property. + +The clause belongs to the engine. This page shows what is specific to Python: a small data set, one statement for each kind of facet, and the dict that comes back. The kinds are `TERMS`, `RANGES`, `HISTOGRAM`, `HIERARCHY`, `DISTINCT` and `STATS`. + +== What you need + +* Engine 26.12.1 or later. An older engine refuses the clause with `SQL syntax error`, because it reads `FACETS` as a table alias, and it refuses `CREATE INDEX ... COLUMNAR` with `Index type 'COLUMNAR' is not supported`. On such an engine no facet statement can run. See <> for the check to make first. +* A `COLUMNAR` index on each property that a facet counts. Creating one, what a close or a crash does to it and its settings are on xref:how-to/columnar-index.adoc[The COLUMNAR index and its settings]. + +The blocks on this page run one after the other against one empty database, and a test in the repository runs them that way (`tests/test_docs_facets_page.py`). Each block that prints something is followed by its output. Every block assumes that `db` is an open database, as on xref:how-to/databases-and-queries.adoc[Databases and queries]. A block with the title "Needs engine 26.12.1 or later" runs `CREATE INDEX ... COLUMNAR` or a `FACETS` query, so an older engine refuses it. The test checks that refusal on an older engine, and it runs and checks the output of the block on a newer one. The other blocks run on any engine. + +== The data + +Ten records with a string, two numbers, a timestamp and a list of strings. The last four records have no `tags`. + +[source,python] +---- +from datetime import datetime + +categories = ["a", "b", "a", "c", "a", "b", "a", "d", "a", "c"] +months = [1, 1, 1, 1, 2, 2, 2, 4, 4, 4] +tags = [["run", "rum"], ["runner"], ["running", "run"], ["rum"], ["run"], ["other"]] +tags += [None] * 4 + +db.command("sql", "CREATE DOCUMENT TYPE FacetDoc") +db.command("sql", "CREATE PROPERTY FacetDoc.id INTEGER") +db.command("sql", "CREATE PROPERTY FacetDoc.category STRING") +db.command("sql", "CREATE PROPERTY FacetDoc.score INTEGER") +db.command("sql", "CREATE PROPERTY FacetDoc.price DOUBLE") +db.command("sql", "CREATE PROPERTY FacetDoc.ts DATETIME") +db.command("sql", "CREATE PROPERTY FacetDoc.tags LIST OF STRING") +with db.transaction(): + for i, category in enumerate(categories): + sql = "INSERT INTO FacetDoc SET id = ?, category = ?, score = ?, price = ?, ts = ?" + args = [i, category, 10 * (i + 1), 0.5 * (i + 1)] + args.append(datetime(2024, months[i], 10 + i, 12, 0, 0)) + if tags[i] is not None: + sql += ", tags = ?" + args.append(tags[i]) + db.command("sql", sql, *args) +print(db.query("sql", "SELECT count(*) AS n FROM FacetDoc").first().get("n")) +---- + +[source,text] +---- +10 +---- + +The index comes after the data here, and it is built over the records that exist. A list needs `BY ITEM`, so that each item of the list is a value. + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +for prop in ("category", "score", "price", "ts"): + db.command("sql", f"CREATE INDEX ON FacetDoc ({prop}) COLUMNAR") +db.command("sql", "CREATE INDEX ON FacetDoc (tags BY ITEM) COLUMNAR") +print(db.schema.exists_index("FacetDoc[category]_columnar")) +---- + +[source,text] +---- +True +---- + +One helper turns a terms facet into `{value: count}`, without the bucket of the records that have no value: + +[source,python] +---- +def counts(facet): + return {b["value"]: b["count"] for b in facet["buckets"] if not b.get("missing")} +---- + +== Terms: the most frequent values + +`TERMS TOP n` returns the `n` values with the most records. A facet has an alias before its property (`s: score`), and when there is none, the alias is the property name. The records of the values left out are counted in `other_count`. + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +rs = db.query( + "sql", "SELECT id, category FROM FacetDoc ORDER BY id FACETS category TERMS TOP 2" +) +rows = rs.to_list() # the rows first, then the facets +facets = rs.facets +print(len(rows), list(facets)) +category = facets["category"] +print( + category["type"], + len(category["buckets"]), + category["buckets"][0], + category["other_count"], + category["missing"], +) +---- + +[source,text] +---- +10 ['meta', 'category'] +terms 2 {'value': 'a', 'count': 5} 3 0 +---- + +Category `a` has 5 records. The second bucket has 2 records, and `b` and `c` both have 2, so which of the two comes second is not fixed. The 3 records of the other values are in `other_count`, and none has a null category (`missing`). + +`SKIP` and `LIMIT` belong to the rows. The facets still count every match: + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id FACETS category TERMS TOP 10 SKIP 2 LIMIT 3", +) +rows = rs.to_list() +print([r["id"] for r in rows]) +print(sorted(counts(rs.facets["category"]).items())) +---- + +[source,text] +---- +[2, 3, 4] +[('a', 5), ('b', 2), ('c', 2), ('d', 1)] +---- + +== Ranges + +`RANGES` counts the records in intervals that you write. `[` includes an end, `)` leaves it out, and `*` is an open end. A bucket has its `key` (the name after `AS`), `from`, `to` and `count`. An open end has no `from` or `to` key. `DEFAULT 'name'` adds a bucket for the records in no interval, and its `default` is `True`. A range facet returns every interval, so it has no `other_count`. + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id " + "FACETS s: score RANGES ([0, 50) AS 'low', [50, *) AS 'high')", +) +rs.to_list() +score = rs.facets["s"] +print(score["type"], [(b["key"], b["count"]) for b in score["buckets"]]) +print("other_count" in score, "to" in score["buckets"][1]) + +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id " + "FACETS s: score RANGES ([0, 30) AS 'low') DEFAULT 'rest'", +) +rs.to_list() +buckets = rs.facets["s"]["buckets"] +print([(b["key"], b["count"], b.get("default", False)) for b in buckets]) +---- + +[source,text] +---- +range [('low', 4), ('high', 6)] +False False +[('low', 2, False), ('rest', 8, True)] +---- + +== Histogram by calendar + +`HISTOGRAM CALENDAR '1M'` makes one bucket for each calendar month of a temporal property. `TIMEZONE` names the time zone of the calendar. The buckets are in time order, and a month with no record (March here) may or may not be listed, so read the buckets with a count. + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id " + "FACETS h: ts HISTOGRAM CALENDAR '1M' TIMEZONE 'UTC'", +) +rs.to_list() +hist = rs.facets["h"] +print(hist["type"], [b["count"] for b in hist["buckets"] if b["count"]]) +---- + +[source,text] +---- +histogram [4, 3, 3] +---- + +== Filter: a facet that ignores its own selection + +A `FILTER` on a facet narrows the rows and the other facets, and the facet that owns the filter ignores it. That is the disjunctive behaviour of a faceted search: after you pick category `a`, the list of categories still shows every category with its count, while the other facets count only what category `a` leaves. + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +rs = db.query( + "sql", + "SELECT id, category FROM FacetDoc ORDER BY id " + "FACETS c: category TERMS TOP 10 FILTER category = 'a', " + "s: score RANGES ([0, 50) AS 'low', [50, *) AS 'high')", +) +rows = rs.to_list() +facets = rs.facets +print([r["id"] for r in rows]) +print(sorted(counts(facets["c"]).items())) +print([(b["key"], b["count"]) for b in facets["s"]["buckets"]]) +---- + +[source,text] +---- +[0, 2, 4, 6, 8] +[('a', 5), ('b', 2), ('c', 2), ('d', 1)] +[('low', 2), ('high', 3)] +---- + +== Stats and distinct counts + +`STATS` returns `count`, `missing`, `min`, `max`, `sum` and `avg` of a numeric property, and `DISTINCT` the number of different values. + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id FACETS s: score STATS, p: price STATS", +) +rs.to_list() +score, price = rs.facets["s"], rs.facets["p"] +print(score["type"], score["count"], score["missing"], score["min"], score["max"], score["sum"]) +print(score["avg"] == 55) +print(price["min"], price["max"], price["sum"], price["avg"]) + +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id " + "FACETS c: category DISTINCT, g: tags DISTINCT", +) +rs.to_list() +facets = rs.facets +print(facets["c"]["value"], facets["c"]["exact"]) +print(facets["g"]["value"], facets["g"]["exact"], facets["g"]["missing"]) +---- + +[source,text] +---- +stats 10 0 10 100 550 +True +0.5 5.0 27.5 2.75 +4 True +5 True 4 +---- + +The `min` and `max` of a temporal property are the `datetime` or `date` that a row holds, and its `sum` and `avg` are `None`. When no record has a value, `min`, `max` and `avg` are `None`. The tags have 5 different values, counted by item, and the 4 records without tags are in `missing`. + +`DISTINCT PRECISION p` keeps the exact set of values of a numeric property while it holds at most `p` values, and switches to an estimate (a sketch) above that. The result then has `exact` set to `False`, and so has `meta["exact"]`. `price` has 10 values: + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id FACETS q: price DISTINCT PRECISION 5", +) +rs.to_list() +q = rs.facets["q"] +print(q["exact"], abs(q["value"] - 10) <= 2, rs.facets["meta"]["exact"]) + +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id FACETS q: price DISTINCT PRECISION 10", +) +rs.to_list() +q = rs.facets["q"] +print(q["value"], q["exact"]) +---- + +[source,text] +---- +False True False +10 True +---- + +With `PRECISION 5` the answer is within 2 of the true 10, and it is flagged as an estimate. With `PRECISION 10` the ten values do not exceed the limit, so the count is exact. A `DISTINCT` without `PRECISION` is exact in the blocks above. + +== Hierarchy + +A `HIERARCHY` facet counts the values of a tag property under their ancestors in a tree of vertices. The index names the tree in its `METADATA`: the vertex type, the property that holds the path of a tag, and the edge type from a parent to a child. The tree here has `t0`, with `t0/a` and `t0/b` under it, and `t0/a/x` under `t0/a`. + +[source,python] +---- +db.command("sql", "CREATE VERTEX TYPE Tag") +db.command("sql", "CREATE PROPERTY Tag.path STRING") +db.command("sql", "CREATE INDEX ON Tag (path) NOTUNIQUE") +db.command("sql", "CREATE EDGE TYPE ParentOf") +db.command("sql", "CREATE DOCUMENT TYPE Tagged") +db.command("sql", "CREATE PROPERTY Tagged.id INTEGER") +db.command("sql", "CREATE PROPERTY Tagged.tags LIST OF STRING") +---- + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +db.command( + "sql", + "CREATE INDEX ON Tagged (tags BY ITEM) COLUMNAR METADATA " + '{"hierarchy": {"vertexType": "Tag", "key": "path", "parentEdge": "ParentOf"}}', +) +with db.transaction(): + for path in ("t0", "t0/a", "t0/a/x", "t0/b"): + db.command("sql", "CREATE VERTEX Tag SET path = ?", path) + for parent, child in (("t0", "t0/a"), ("t0", "t0/b"), ("t0/a", "t0/a/x")): + db.command( + "sql", + "CREATE EDGE ParentOf FROM (SELECT FROM Tag WHERE path = ?) " + "TO (SELECT FROM Tag WHERE path = ?)", + parent, + child, + ) + records = [ + (1, ["t0/a/x"]), + (2, ["t0/a"]), + (3, ["t0/b", "t0/a/x"]), + (4, ["orphan"]), # a value no Tag vertex has + (5, None), # no value at all + ] + for record_id, record_tags in records: + if record_tags is None: + db.command("sql", "INSERT INTO Tagged SET id = ?", record_id) + else: + db.command( + "sql", + "INSERT INTO Tagged SET id = ?, tags = ?", + record_id, + record_tags, + ) + +rs = db.query( + "sql", "SELECT id FROM Tagged ORDER BY id FACETS t: tags HIERARCHY TOP 100" +) +rs.to_list() +tree = rs.facets["t"] +print(tree["type"], tree["other_count"], tree["missing"]) +by_value = {b["value"]: b for b in tree["buckets"] if not b.get("missing")} +for value, b in sorted(by_value.items()): + print(value, b["count"], b.get("parent"), b["depth"], b.get("orphan", False)) +---- + +[source,text] +---- +hierarchy 0 1 +orphan 1 None 0 True +t0 3 None 0 False +t0/a 3 t0 1 False +t0/a/x 2 t0/a 2 False +t0/b 1 t0 1 False +---- + +The buckets are a flat list, not a nested one. A bucket has the `value` (the path of the tag), `count`, `parent` (the value of its first parent, `None` for a root) and `depth`. Each record counts once under each ancestor of its tags: record 3 has `t0/b` and `t0/a/x`, which share `t0`, and it adds one to `t0`. A value that no vertex of the tree has, here `orphan`, is its own root, and its `orphan` is `True`. With `TOP n`, `other_count` is the sum of the counts of the buckets left out, and those counts overlap, so it can exceed the number of matches. + +== Paging through the values + +A terms facet sorted by value pages with a cursor. The first page says `ORDER BY VALUE`. Every later page says `AFTER` and the `after` of the page before, until `after` is `None`. `after` is the last value of a full page, as a row holds it, so it can be a string, a number or a datetime. A facet that is not sorted by value has no `after`. The records without a value come once, on the last page, as a bucket with `MISSING` after `TOP n`. The cursor can be named, as here, or positional (`AFTER ?`). + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +cursor = None +page = 0 +while True: + page += 1 + if cursor is None: + sql = "SELECT id FROM FacetDoc FACETS f: category TERMS TOP 2 ORDER BY VALUE" + rs = db.query("sql", sql) + else: + sql = "SELECT id FROM FacetDoc FACETS f: category TERMS TOP 2 AFTER :c" + rs = db.query("sql", sql, {"c": cursor}) + rs.to_list() + facet = rs.facets["f"] + print(page, [(b["value"], b["count"]) for b in facet["buckets"]], facet["after"]) + cursor = facet["after"] + if cursor is None: + break +---- + +[source,text] +---- +1 [('a', 5), ('b', 2)] b +2 [('c', 2), ('d', 1)] None +---- + +`ORDER BY COUNT` cannot be paged: the engine refuses `AFTER` with it (`AFTER pages in value order`). + +== Typeahead: facet.searchValues + +`facet.searchValues(type, property, prefix, limit)` is an ordinary SQL function. It returns the values of a `COLUMNAR` string property (or a list of strings indexed `BY ITEM`) that start with a prefix, in value order, as rows with a `value` and the `count` of the live records that carry it. The prefix is literal, never a pattern. A value that no record holds any more is not suggested. + +The function reads the values that the projection has applied, and the engine applies them in the background. Right after a load it can return fewer values, so the block below first counts the tags with a terms facet, which needs no wait, and then tries the search again until it gets what it expects. The limit of 120 seconds is only there so that a stuck engine cannot hang the block. + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +import time + +rs = db.query( + "sql", "SELECT id FROM FacetDoc ORDER BY id FACETS g: tags TERMS TOP 10" +) +rs.to_list() +print(sorted(counts(rs.facets["g"]).items())) + + +def suggest(prefix, limit): + rows = db.query( + "sql", + "SELECT expand(facet.searchValues(?, ?, ?, ?))", + "FacetDoc", + "tags", + prefix, + limit, + ).to_list() + return [(r["value"], r["count"]) for r in rows] + + +def suggest_when_ready(prefix, limit, expected, seconds=120): + deadline = time.monotonic() + seconds + while True: + try: + found = suggest(prefix, limit) + if found == expected or time.monotonic() > deadline: + return found + except Exception: # the engine asks for a retry while a column builds + if time.monotonic() > deadline: + raise + time.sleep(0.05) + + +print(suggest_when_ready("run", 20, [("run", 3), ("runner", 1), ("running", 1)])) +print(suggest("ru", 2)) +print(suggest("zzz", 20)) +---- + +[source,text] +---- +[('other', 1), ('rum', 2), ('run', 3), ('runner', 1), ('running', 1)] +[('run', 3), ('runner', 1), ('running', 1)] +[('rum', 2), ('run', 3)] +[] +---- + +== Reading facets in Python + +`ResultSet.facets` is a plain dict, the same on `arcadedb-jvm` and on the native `arcadedb`. It is `None` for a statement without a `FACETS` clause. It works on the result of `db.query(...)` and of `db.command(...)`. Reading it from a result set that was closed before its end raises `ArcadeDBError`. + +`meta` comes first. Every other key is a facet, by its alias, in the order of the clause. + +[cols="2,5"] +|=== +| Key | Meaning + +| `meta` +| `exact` (`False` when the counts may lag the committed data, and for an estimate), `patched` (records that were read from the record itself to correct the projection) and `lag_ms`. The counts cover the records that the projection has not applied yet, by patching them, so a facet query needs no wait. + +| `type` +| `terms`, `range`, `histogram`, `hierarchy`, `distinct` or `stats`. + +| `source` +| `index` (answered from the COLUMNAR index), `scan` (from the records) or `unavailable`. + +| `reason` +| Why a facet is unavailable. A facet that the engine answered has none. + +| `buckets` +| Terms, hierarchy, range and histogram facets. A terms bucket has `value` and `count`. A range or histogram bucket has `key`, `from`, `to` and `count`, and an open end has no `from` or `to`. A bucket of the values in no range has `key` and `default`. The bucket of the records without a value has `value` `None` and `missing` `True`. + +| `other_count` +| Terms and hierarchy facets: the records of the values that `TOP` left out. + +| `missing` +| Every facet except `stats`: the records without a value. + +| `count`, `min`, `max`, `sum`, `avg` +| A stats facet, with `missing`. The values are the Python values a row holds, so NaN and the infinities are floats. + +| `value`, `exact` +| A distinct facet. + +| `after` +| A terms facet sorted by value: the cursor of the next page, or `None` on the last page. + +| `parent`, `depth`, `orphan` +| The extra keys of a hierarchy bucket. +|=== + +=== The rows first + +The engine computes the facets when the rows are exhausted, so read the rows first, as in every block above, and the facets after. Reading `facets` before the rows gives the same facets, but it makes the engine count the matches that the rows did not reach, and whether the rows that were not fetched yet can still be read after that is not specified. The facets stay available after the rows are read. + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +rs = db.query( + "sql", "SELECT id FROM FacetDoc ORDER BY id FACETS category TERMS TOP 2" +) +seen = [row.get("id") for row in rs] +facets = rs.facets +print(seen == list(range(10)), facets["category"]["buckets"][0]["count"]) +print(rs.facets == facets, rs.to_list()) +print(db.query("sql", "SELECT FROM FacetDoc").facets is None) +---- + +[source,text] +---- +True 5 +True [] +True +---- + +=== Unavailable facets + +A facet that the engine cannot answer is kept, with its `type`, its `source` set to `unavailable` and a `reason`, and nothing else. The rows and the other facets of the same query are answered. The example is a `HIERARCHY` facet on a type whose subtype alone has the hierarchy: the records of `Media` have none. + +[source,python] +---- +db.command("sql", "CREATE VERTEX TYPE Label") +db.command("sql", "CREATE PROPERTY Label.path STRING") +db.command("sql", "CREATE INDEX ON Label (path) NOTUNIQUE") +db.command("sql", "CREATE EDGE TYPE LabelParent") +db.command("sql", "CREATE DOCUMENT TYPE Media") +db.command("sql", "CREATE PROPERTY Media.tags LIST OF STRING") +db.command("sql", "CREATE DOCUMENT TYPE Photo EXTENDS Media") +---- + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +db.command( + "sql", + "CREATE INDEX ON Photo (tags BY ITEM) COLUMNAR METADATA " + '{"hierarchy": {"vertexType": "Label", "key": "path", "parentEdge": "LabelParent"}}', +) +with db.transaction(): + db.command("sql", "CREATE VERTEX Label SET path = 't0'") + db.command("sql", "CREATE VERTEX Label SET path = 't0/t1'") + db.command( + "sql", + "CREATE EDGE LabelParent FROM (SELECT FROM Label WHERE path = 't0') " + "TO (SELECT FROM Label WHERE path = 't0/t1')", + ) + for _ in range(3): + db.command("sql", "INSERT INTO Media SET tags = ['t0/t1']") + db.command("sql", "INSERT INTO Photo SET tags = ['t0/t1']") + +rs = db.query( + "sql", + "SELECT FROM Media FACETS t: tags HIERARCHY TOP 100, n: tags TERMS TOP 5", +) +rows = rs.to_list() +facets = rs.facets +tree = facets["t"] +print(len(rows), tree["type"], tree["source"], sorted(tree)) +print("Media.tags" in tree["reason"]) +print(counts(facets["n"])) +---- + +[source,text] +---- +6 hierarchy unavailable ['reason', 'source', 'type'] +True +{'t0/t1': 6} +---- + +== Parameters + +The bounds of `RANGES` and `HISTOGRAM`, the values of `INCLUDE`, the cursor of `AFTER` and the condition of `FILTER` take parameters, named (`:lo`) or positional (`?`), through `db.query` and `db.command`. `TOP` and `PRECISION` take an integer literal, not a parameter. + +.Needs engine 26.12.1 or later +[source,python,role=needs-engine] +---- +named = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id FACETS s: score RANGES ([:lo, :hi) AS 'mid')", + {"lo": 30, "hi": 70}, +) +named.to_list() +positional = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id FACETS s: score RANGES ([?, ?) AS 'mid')", + 30, + 70, +) +positional.to_list() +for rs in (named, positional): + (mid,) = rs.facets["s"]["buckets"] + print(mid["key"], mid["count"]) + +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id FACETS c: category TERMS INCLUDE [:x, :y]", + {"x": "a", "y": "c"}, +) +rs.to_list() +print(sorted(counts(rs.facets["c"]).items())) +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id FACETS c: category TERMS INCLUDE [?, ?]", + "a", + "c", +) +rs.to_list() +print(sorted(counts(rs.facets["c"]).items())) + +rs = db.query( + "sql", + "SELECT id FROM FacetDoc ORDER BY id " + "FACETS c: category DISTINCT, s: score STATS FILTER score >= :lo", + {"lo": 60}, +) +rows = rs.to_list() +print([r["id"] for r in rows]) +print(rs.facets["c"]["value"], rs.facets["c"]["exact"]) +---- + +[source,text] +---- +mid 4 +mid 4 +[('a', 5), ('c', 2)] +[('a', 5), ('c', 2)] +[5, 6, 7, 8, 9] +4 True +---- + +The `[30, 70)` interval holds the scores 30, 40, 50 and 60. The last query shows that the filter of the `STATS` facet narrows the rows, and the `DISTINCT` facet counts the 4 categories of the records that pass. + +[[older-engine]] +== An older engine + +An engine before 26.12.1 refuses the `COLUMNAR` index and the `FACETS` clause, as described at the top. The engine tells its version through SQL, so check it before you run a facet statement, on either package: + +[source,python] +---- +def engine_version(db): + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + parts = [ + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ] + return tuple(parts + [0] * (3 - len(parts))) + + +has_facets = engine_version(db) >= (26, 12, 1) +---- + +A snapshot or a build suffix of the version is dropped, so `26.12.1-SNAPSHOT` counts as 26.12.1. The https://github.com/ArcadeData/arcadedb-python/blob/main/examples/30_columnar_facets.py[example] makes this check, and prints one line and stops when the engine is too old. + +== Known limits + +* The extremes of a negative zero come back as `0.0`, and `0.0 == -0.0` in Python, so a `MIN` or `MAX` facet cannot tell them apart. A `DISTINCT` counts `-0.0` and `0.0` as one value. +* A `DISTINCT` with `PRECISION p` is an estimate once the property has more than `p` values, and it says so with `exact` set to `False`. +* `facet.searchValues` can lag right after a load, as described above. +* Facets count properties that have a `COLUMNAR` index. After the index is dropped, `FACETS` answers from a scan of the records (`source` is `scan`). A facet that the engine cannot answer at all is `unavailable`, with its reason. + +== Where to go next + +* xref:how-to/columnar-index.adoc[The COLUMNAR index and its settings]: creating and dropping the index, what a close or a crash does to it, the `arcadedb.columnar.*` settings and the acceleration of `GROUP BY`. +* xref:how-to/databases-and-queries.adoc[Databases and queries]: `db.query`, `db.command`, parameters and result sets. +* `examples/30_columnar_facets.py` in the repository runs a faceted query over a larger data set and checks every count against Python. diff --git a/examples/30_columnar_facets.py b/examples/30_columnar_facets.py new file mode 100644 index 0000000..fc9ccea --- /dev/null +++ b/examples/30_columnar_facets.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Example 30: Faceted Queries with a COLUMNAR Index. + +A product list with the numbers of a search sidebar next to it: the rows of a +query (a page of five), and the counts of the same query for each category, +each price range and each month, over ALL the matches, not only the five rows. +The counts come from the ``FACETS`` clause of a ``SELECT``, and Python reads +them from ``result_set.facets``, a plain dict, the same on ``arcadedb`` and on +``arcadedb-jvm``. + +Workflow covered: +- create a document type, load it, then ``CREATE INDEX ... COLUMNAR`` on the + properties that the facets count +- one query with a ``TERMS`` facet (the 3 biggest categories) and a ``RANGES`` + facet (price bands), with a page of rows: ``LIMIT 5`` +- one query with ``STATS`` (count, min, max, sum, average of the price) and + ``DISTINCT`` (number of brands) +- one query with a calendar ``HISTOGRAM`` (one bucket for each month) +- every count is checked against the same count made in Python over the rows + that were inserted, so the example fails when a facet is wrong + +Requirements: +- ArcadeDB engine 26.12.1 or later. On an older engine the example prints one + line and exits with status 0: the engine refuses the ``FACETS`` clause and the + ``COLUMNAR`` index type there. + +Notes: +- Read the rows first and ``result_set.facets`` after, as the example does. +- No data download: the products are generated from the row number. +""" + +from __future__ import annotations + +import argparse +import shutil +from collections import Counter +from datetime import datetime +from pathlib import Path + +import arcadedb + +FACETS_SINCE = (26, 12, 1) + +CATEGORIES = ["laptop", "phone", "tablet", "monitor", "keyboard", "mouse"] +BRANDS = ["Acme", "Globex", "Initech", "Umbrella", "Hooli"] +MIN_PRICE = 20 # the WHERE of every query below + +TERMS_AND_RANGES = ( + f"SELECT id, name, price FROM Product WHERE price >= {MIN_PRICE} ORDER BY id " + "FACETS p: price RANGES ([0, 100) AS 'low', [100, 300) AS 'mid', [300, *) AS 'high'), " + "c: category TERMS TOP 3 LIMIT 5" +) +STATS_AND_DISTINCT = ( + f"SELECT id FROM Product WHERE price >= {MIN_PRICE} ORDER BY id " + "FACETS s: price STATS, d: brand DISTINCT" +) +MONTHLY = ( + f"SELECT id FROM Product WHERE price >= {MIN_PRICE} ORDER BY id " + "FACETS h: created HISTOGRAM CALENDAR '1M' TIMEZONE 'UTC'" +) + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--rows", type=int, default=1200, help="number of products") + p.add_argument("--db-path", default="./my_test_databases/columnar_facets") + return p.parse_args() + + +def engine_version(db) -> tuple[str, tuple[int, int, int]]: + """The engine's version text and its x.y.z (a SNAPSHOT or a build suffix is dropped).""" + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + parts = [ + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ] + return raw, tuple(parts + [0] * (3 - len(parts))) + + +def make_products(count: int) -> list[dict]: + """Deterministic products: nothing is random, so every run has the same counts.""" + return [ + { + "id": i, + "name": f"item-{i:04d}", + "category": CATEGORIES[(i + i // 4) % len(CATEGORIES)], + "brand": BRANDS[(i * 3 + i // 7) % len(BRANDS)], + "price": 5 + (i * 37) % 495, + # noon on days 1 to 27, so no time zone moves a record to another month + "created": datetime(2024, 1 + i % 12, 1 + (i * 5) % 27, 12, 0, 0), + } + for i in range(count) + ] + + +def load(db, products: list[dict]) -> None: + for ddl in ( + "CREATE DOCUMENT TYPE Product", + "CREATE PROPERTY Product.id INTEGER", + "CREATE PROPERTY Product.name STRING", + "CREATE PROPERTY Product.category STRING", + "CREATE PROPERTY Product.brand STRING", + "CREATE PROPERTY Product.price INTEGER", + "CREATE PROPERTY Product.created DATETIME", + ): + db.command("sql", ddl) + with db.transaction(): + for p in products: + db.command( + "sql", + "INSERT INTO Product SET id = ?, name = ?, category = ?, brand = ?, " + "price = ?, created = ?", + p["id"], + p["name"], + p["category"], + p["brand"], + p["price"], + p["created"], + ) + # the index is built over the records that exist, and kept up to date after + for prop in ("category", "brand", "price", "created"): + db.command("sql", f"CREATE INDEX ON Product ({prop}) COLUMNAR") + + +def show_terms_and_ranges(db, matches: list[dict]) -> None: + rs = db.query("sql", TERMS_AND_RANGES) + rows = rs.to_list() # the rows first, then the facets + facets = rs.facets + print(f"\nrows (page of {len(rows)} of {len(matches)} matches):") + for r in rows: + print(f" {r['id']:>4} {r['name']} {r['price']:>4}") + assert [r["id"] for r in rows] == [p["id"] for p in matches[:5]] + + print(f"\nfacets, over all {len(matches)} matches, not the {len(rows)} rows:") + categories = facets["c"] + print(f" category ({categories['type']}, from {categories['source']}):") + for b in categories["buckets"]: + print(f" {b['value']:<10} {b['count']:>5}") + print(f" {'(other)':<10} {categories['other_count']:>5}") + by_category = Counter(p["category"] for p in matches) + assert len(categories["buckets"]) == 3 + for b in categories["buckets"]: + assert b["count"] == by_category[b["value"]], b + # ties of the same count may come in either order, so compare the counts, not the names + assert [b["count"] for b in categories["buckets"]] == sorted( + by_category.values(), reverse=True + )[:3] + assert categories["other_count"] == len(matches) - sum( + b["count"] for b in categories["buckets"] + ) + + prices = facets["p"] + print(f" price ({prices['type']}):") + for b in prices["buckets"]: + print(f" {b['key']:<10} {b['count']:>5}") + expected = Counter( + "low" if p["price"] < 100 else "mid" if p["price"] < 300 else "high" + for p in matches + ) + assert {b["key"]: b["count"] for b in prices["buckets"]} == dict(expected) + meta = facets["meta"] + print(f" meta: exact={meta['exact']} patched={meta['patched']}") + + +def show_stats_and_distinct(db, matches: list[dict]) -> None: + rs = db.query("sql", STATS_AND_DISTINCT) + assert len(rs.to_list()) == len(matches) + facets = rs.facets + stats, brands = facets["s"], facets["d"] + print("\nstats and distinct:") + print( + f" price: count={stats['count']} min={stats['min']} max={stats['max']} " + f"sum={stats['sum']} avg={stats['avg']:.2f}" + ) + print(f" brands: {brands['value']} (exact: {brands['exact']})") + prices = [p["price"] for p in matches] + assert (stats["count"], stats["missing"]) == (len(prices), 0) + assert (stats["min"], stats["max"], stats["sum"]) == ( + min(prices), + max(prices), + sum(prices), + ) + assert abs(stats["avg"] - sum(prices) / len(prices)) < 1e-9 + assert (brands["value"], brands["exact"]) == ( + len({p["brand"] for p in matches}), + True, + ) + + +def show_monthly(db, matches: list[dict]) -> None: + rs = db.query("sql", MONTHLY) + assert len(rs.to_list()) == len(matches) + hist = rs.facets["h"] + print("\nproducts for each month of 2024:") + full = [ + b for b in hist["buckets"] if b["count"] + ] # a month with no record may be listed + for b in full: + print(f" {b['key']:<10} {b['count']:>5}") + by_month = Counter(p["created"].month for p in matches) + assert [b["count"] for b in full] == [by_month[m] for m in sorted(by_month)] + + +def main() -> None: + args = parse_args() + db_path = Path(args.db_path) + if db_path.exists(): + shutil.rmtree(db_path) + db_path.parent.mkdir(parents=True, exist_ok=True) + with arcadedb.create_database(str(db_path)) as db: + raw_version, version = engine_version(db) + if version < FACETS_SINCE: + print( + f"Engine {raw_version} has no FACETS clause and no COLUMNAR index " + "(they need engine 26.12.1 or later): nothing to run." + ) + return + print(f"engine: arcadedb {arcadedb.__version__}, engine version {raw_version}") + + products = make_products(args.rows) + load(db, products) + matches = [p for p in products if p["price"] >= MIN_PRICE] + print( + f"loaded {len(products)} products, {len(matches)} with price >= {MIN_PRICE}" + ) + + show_terms_and_ranges(db, matches) + show_stats_and_distinct(db, matches) + show_monthly(db, matches) + print("\nall facet counts match the counts made in Python") + + +if __name__ == "__main__": + main() diff --git a/examples/README.md b/examples/README.md index 46303d3..e3f0c06 100644 --- a/examples/README.md +++ b/examples/README.md @@ -441,6 +441,24 @@ full-text and graph context in a single query --- +### 🔎 [30_columnar_facets.py](./30_columnar_facets.py) +**Faceted Queries | COLUMNAR Index | `FACETS` Clause | `result_set.facets`** + +The counts of a search sidebar next to the rows of a query: +- `CREATE INDEX ... COLUMNAR` on the properties that the facets count +- one query with a `TERMS` facet (the 3 biggest categories) and a `RANGES` facet (price + bands) next to a page of 5 rows, then `STATS` with `DISTINCT`, then a calendar `HISTOGRAM` +- the facets cover all the matches of the `WHERE`, not only the rows of the page +- checks every count against the same count made in Python over the inserted rows +- needs engine 26.12.1 or later: on an older engine it prints one line and exits with + status 0 +- generated data, no download; flags: `--rows` (default 1,200) and `--db-path` + +**Learn:** How to ask for counts per value, range and month in the same statement as the +rows, and how to read them from the result set as a dict + +--- + ## 💡 Tips - **Run from examples/ directory** - Always execute examples from `examples/` for correct file paths diff --git a/tests/test_docs_facets_page.py b/tests/test_docs_facets_page.py new file mode 100644 index 0000000..fa07d2c --- /dev/null +++ b/tests/test_docs_facets_page.py @@ -0,0 +1,120 @@ +"""Every Python block of the Antora page ``docs/modules/ROOT/pages/how-to/facets.adoc`` runs, and prints what the page says. + +The blocks of the page are chained on purpose: they run in the order of the page, in one namespace, against one fresh temporary +database that the namespace holds as ``db`` (the page says that each block assumes an open ``db``). A block that is followed by a +``[source,text]`` block must print exactly that text. The test uses no Java object, so it runs on the JVM build and the native +build alike. + +Facets are new in engine 26.12.1 (see tests/test_columnar_engine_smoke.py). The engine of a lane is whatever the lane links: the +JVM lane runs the ``latest`` image, the native pull request gate links 26.11.1. A block of the page that creates a COLUMNAR index +or runs a ``FACETS`` query carries ``role=needs-engine`` and the title "Needs engine 26.12.1 or later". On a newer engine such a +block runs and prints what the page shows. On an older engine the same block must be refused with the engine's own error, which +is what the page documents, and the test asserts that refusal instead of the output. Nothing is skipped (the repository accepts no +skips, scripts/check_test_skips.py). The version is read through the test's database (``SELECT version()``), so the same check +runs on both builds. +""" + +import contextlib +import io +import re +from pathlib import Path + +import pytest + +FACETS_SINCE = (26, 12, 1) + +PAGE = ( + Path(__file__).resolve().parents[1] + / "docs" + / "modules" + / "ROOT" + / "pages" + / "how-to" + / "facets.adoc" +) + +# a Python listing (its header may name a role, and a title line may come before it), then (optionally, directly after it) the +# text listing with its output +BLOCK_RE = re.compile( + r"(?:^(?P\.[^\n]*)\n)?" + r"\[source,python(?P<attrs>[^\]\n]*)\]\n----\n(?P<code>.*?)\n----\n" + r"(?:\n\[source,text\]\n----\n(?P<out>.*?)\n----\n)?", + re.DOTALL | re.MULTILINE, +) + +NEEDS_TITLE = ".Needs engine 26.12.1 or later" + +# what an engine before 26.12.1 answers: a FACETS clause is a syntax error, a COLUMNAR index is an unknown index type +REFUSAL = r"SQL syntax error|Index type 'COLUMNAR' is not supported|COLUMNAR" + + +def _blocks(): + """The Python blocks as (code, expected output or None, needs the new engine).""" + text = PAGE.read_text(encoding="utf-8") + found = [] + for m in BLOCK_RE.finditer(text): + needs = "role=needs-engine" in m.group("attrs") + # the title and the role say the same thing, in both directions + assert needs == (m.group("title") == NEEDS_TITLE), m.group("code")[:80] + found.append((m.group("code"), m.group("out"), needs)) + # no Python listing may escape the extraction (a different spelling of the block header would run nowhere) + assert text.count("[source,python") == len(found) + return found + + +def _engine_version(db): + """The engine's x.y.z through SQL `SELECT version()`; a SNAPSHOT or a build suffix is dropped.""" + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + parts = [ + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ] + return tuple(parts + [0] * (3 - len(parts))) + + +def test_the_page_has_its_blocks(): + blocks = _blocks() + assert len(blocks) >= 18 + assert sum(1 for _, out, _ in blocks if out is not None) >= 14 + needing = [code for code, _, needs in blocks if needs] + assert len(needing) >= 12 + # every block that needs the new engine says so: its first engine call is a COLUMNAR index or a FACETS query + for code in needing: + assert "COLUMNAR" in code or "FACETS" in code, code[:80] + # and no other block does: they all run on an engine without facets + for code, _, needs in blocks: + if not needs: + assert "FACETS" not in code and "COLUMNAR" not in code, code[:80] + + +def test_every_block_runs_in_order_and_prints_what_the_page_says(temp_db): + namespace = {"db": temp_db, "__name__": "facets_page"} + newer = _engine_version(temp_db) >= FACETS_SINCE + refused = 0 + for number, (code, expected, needs) in enumerate(_blocks(), start=1): + compiled = compile(code, f"facets.adoc python block {number}", "exec") + captured = io.StringIO() + if needs and not newer: + # an older engine: the block is refused with the engine's own error, before it prints anything + with pytest.raises(Exception, match=REFUSAL): + with contextlib.redirect_stdout(captured): + exec( + compiled, namespace + ) # nosec B102 - the page of this repository + assert captured.getvalue() == "", f"block {number} printed before it failed" + refused += 1 + continue + with contextlib.redirect_stdout(captured): + exec(compiled, namespace) # nosec B102 - the page of this repository + if expected is not None: + assert ( + captured.getvalue().strip() == expected.strip() + ), f"block {number} printed other text than the page shows:\n{code}" + else: + assert ( + captured.getvalue() == "" + ), f"block {number} prints, the page shows no output" + + # the page's own version check agrees with the test's, on the engine of this lane + assert namespace["has_facets"] is newer + assert (refused == 0) == newer