diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 63c475123..d6d11f326 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2100,10 +2100,12 @@ native_reclaim native_reclaim reused-block data matches heap mirror 15;16;17;18; native_reclaim native_reclaim row count correct after cycle 15;16;17;18;19 never - native_reclaim native_reclaim ten row groups, every one half deleted 15;16;17;18;19 2026-10-14 the write state ignores stripe_row_limit objstore_credentials objstore_credentials a mapping with an access key and no secret is refused before any request (28000) 15;16;17;18;19 2026-10-14 the user-mapping credential pairing check removed (an access key alone crashes the backend) +objstore_credentials objstore_credentials a role with no mapping of its own reads through the PUBLIC mapping 15;16;17;18;19 2026-10-14 the user-mapping lookup has no PUBLIC fallback +objstore_credentials objstore_credentials a role's own mapping is used before the PUBLIC one (bob's wrong secret is still refused 28000) 15;16;17;18;19 2026-10-14 the user-mapping lookup tries PUBLIC before the role's own objstore_credentials objstore_credentials ambient-is-privilege: carol (credentials_required=false, but no env exists) is 28000 15;16;17;18;19 never - objstore_credentials objstore_credentials ambient: a superuser with no mapping reads via the environment 15;16;17;18;19 never - objstore_credentials objstore_credentials ambient: carol (credentials_required=false) now reads via the environment 15;16;17;18;19 never - -objstore_credentials objstore_credentials ambient: without her mapping, carol is refused despite the environment 15;16;17;18;19 never - +objstore_credentials objstore_credentials ambient: without her mapping, carol is refused despite the environment 15;16;17;18;19 2026-10-14 every mapping may use the server's ambient credentials (allow_ambient = true) objstore_credentials objstore_credentials and neither half-credential read reached the endpoint 15;16;17;18;19 never - objstore_credentials objstore_credentials and one with a secret and no access key is refused the same way 15;16;17;18;19 2026-10-14 the user-mapping credential pairing check removed (an access key alone crashes the backend) objstore_credentials objstore_credentials catalog: count(*) with catalog credentials only 15;16;17;18;19 never - @@ -2115,7 +2117,7 @@ objstore_credentials objstore_credentials no-leak: srvoptions carry no secret (b objstore_credentials objstore_credentials per-user: alice (good secret in her mapping) reads 15;16;17;18;19 never - objstore_credentials objstore_credentials per-user: bob (wrong secret in his mapping) is refused 28000 15;16;17;18;19 never - objstore_credentials objstore_credentials placement: a credential on a USER MAPPING is accepted 15;16;17;18;19 never - -objstore_credentials objstore_credentials placement: a non-superuser cannot set credentials_required false (42501) 15;16;17;18;19 never - +objstore_credentials objstore_credentials placement: a non-superuser cannot set credentials_required false (42501) 15;16;17;18;19 2026-10-14 the validator lets a non-superuser set credentials_required false objstore_credentials objstore_credentials placement: a superuser can (for carol, used below) 15;16;17;18;19 never - objstore_credentials objstore_credentials placement: endpoint and region on the SERVER are accepted 15;16;17;18;19 never - objstore_credentials objstore_credentials placement: endpoint on a TABLE is HV00D 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 94318cae3..ca9714272 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2150,4 +2150,27 @@ suites_not_covered 237 # never 2041 + 27 - 18 = 2050 # dated 615 + 18 = 633 # 2050 + 633 == 2683 -checks_never_observed_red 2050 +# +# #1341, objstore_credentials gets its pytest twin (OffgridwithJD): 2050 -> 2048. No new +# rows. Two arms are dated for the first time: every mapping allowed the ambient +# credentials (carol then read through the environment without her mapping), and the +# validator letting a non-superuser set credentials_required false. Two shell arms +# that expected an empty SQLSTATE now expect "ok", so an empty capture cannot pass. +# Five-major runs: 25/25 on 15-19, the twin passing with pyarrow required. +# +# ledger rows 2683 = 2683 +# never 2050 - 2 = 2048 +# dated 633 + 2 = 635 +# 2048 + 635 == 2683 +# +# #1341, a PUBLIC user mapping (OffgridwithJD): 2048 -> 2048. Two new objstore_credentials +# arms, each dated: the lookup without its PUBLIC fallback (a role with no mapping of +# its own then read nothing), and the lookup trying PUBLIC first (bob's own wrong +# secret was then bypassed by the PUBLIC mapping's good one). Five-major runs: 27/27 +# on 15-19, the twin passing with pyarrow required. +# +# ledger rows 2683 + 2 = 2685 +# never 2048 = 2048 +# dated 635 + 2 = 637 +# 2048 + 637 == 2685 +checks_never_observed_red 2048 diff --git a/test/objstore_credentials.sh b/test/objstore_credentials.sh index d6ebc40fc..1b9baed98 100755 --- a/test/objstore_credentials.sh +++ b/test/objstore_credentials.sh @@ -55,12 +55,21 @@ pg_restart_env() { echo "FATAL: cluster did not come back after env restart"; exit 1 } -sqlstate_of() { # sqlstate_of [role] - env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U "${2:-postgres}" \ - -d "$PGC_DB" -qtA 2>&1 < [role] -> the SQLSTATE the statement fails with; "ok" only when +# psql ran it and exited 0; "noconnect" when there was no session to run it in. Success +# is positive evidence (#1400 review): an empty capture, which a session that never +# connected also gives, is never read as success. +sqlstate_of() { + local out rc st + out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U "${2:-postgres}" \ + -d "$PGC_DB" -qtA -v ON_ERROR_STOP=1 2>&1 < env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U "$1" \ @@ -112,7 +121,7 @@ check "placement: endpoint and region on the SERVER are accepted" \ "$(q "SELECT count(*) FROM pg_foreign_server WHERE srvname = 'csrv'")" "1" check "placement: a credential on a USER MAPPING is accepted" \ "$(sqlstate_of "CREATE USER MAPPING FOR postgres SERVER csrv - OPTIONS (access_key_id '$AKID', secret_access_key '$SECRET')")" "" + OPTIONS (access_key_id '$AKID', secret_access_key '$SECRET')")" "ok" check "placement: endpoint on a USER MAPPING is HV00D" \ "$(sqlstate_of "CREATE USER MAPPING FOR cred_alice SERVER csrv OPTIONS (endpoint 'x')")" "HV00D" @@ -125,7 +134,7 @@ psql_run "CREATE USER MAPPING FOR cred_bob SERVER csrv check "placement: a non-superuser cannot set credentials_required false (42501)" \ "$(sqlstate_of "ALTER USER MAPPING FOR cred_alice SERVER csrv OPTIONS (ADD credentials_required 'false')" cred_alice)" "42501" check "placement: a superuser can (for carol, used below)" \ - "$(sqlstate_of "CREATE USER MAPPING FOR cred_carol SERVER csrv OPTIONS (credentials_required 'false')")" "" + "$(sqlstate_of "CREATE USER MAPPING FOR cred_carol SERVER csrv OPTIONS (credentials_required 'false')")" "ok" # ---- tables ----------------------------------------------------------------- psql_run "CREATE FOREIGN TABLE flocal ($COLDEFS) SERVER csrv OPTIONS (path '$PGC_WORKDIR/$BUCKET/fixture.parquet');" @@ -209,6 +218,22 @@ check_num "no-leak: EXPLAIN VERBOSE carries no secret" \ check_num "no-leak: srvoptions carry no secret (by construction)" \ "$(q "SELECT count(*) FROM pg_foreign_server WHERE srvoptions::text LIKE '%$SECRET%'")" "0" +# ---- a PUBLIC mapping (#1341) ---------------------------------------------- +# sql-reference.md: a read resolves the scanning user's own mapping first, then a +# PUBLIC one. Frank has no mapping of his own; bob's own carries the wrong secret, so +# a PUBLIC mapping holding the right one must not rescue him. Dropped again before +# phase 2, where a PUBLIC mapping would answer for the ambient arms. +psql_run "CREATE ROLE cred_frank LOGIN; GRANT pg_read_server_files TO cred_frank; + GRANT USAGE ON FOREIGN SERVER csrv TO cred_frank; GRANT SELECT ON fs3 TO cred_frank;" +psql_run "CREATE USER MAPPING FOR PUBLIC SERVER csrv + OPTIONS (access_key_id '$AKID', secret_access_key '$SECRET');" +# A refused read counts no rows, so a refusal is a measurement and not an empty capture. +check_num "a role with no mapping of its own reads through the PUBLIC mapping" \ + "$(c="$(q_as cred_frank "SELECT count(*) FROM fs3")"; echo "${c:-0}")" "$((G * ROWS_PER_GROUP))" +check "a role's own mapping is used before the PUBLIC one (bob's wrong secret is still refused 28000)" \ + "$(sqlstate_of "SELECT count(*) FROM fs3" cred_bob)" "28000" +psql_run "DROP USER MAPPING FOR PUBLIC SERVER csrv;" + # ---- phase 2: environment present; ambient is still a privilege ------------- pg_restart_env "AWS_ENDPOINT_URL='http://127.0.0.1:$S3_PORT'" \ "AWS_ACCESS_KEY_ID='$AKID'" "AWS_SECRET_ACCESS_KEY='$SECRET'" \ diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 603fb4b75..82029460f 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -156,6 +156,7 @@ behaviour, the source of that number is named. - [108. test_arrow_import.py: import_arrow round trips, reads a foreign file, and refuses what it cannot decode](#108-test_arrow_importpy-import_arrow-round-trips-reads-a-foreign-file-and-refuses-what-it-cannot-decode) - [109. test_entry_point_privilege.py: every relation-taking entry point has a decided privilege bar](#109-test_entry_point_privilegepy-every-relation-taking-entry-point-has-a-decided-privilege-bar) - [110. test_am_contracts.py: contracts the access method owes core DML and transactions](#110-test_am_contractspy-contracts-the-access-method-owes-core-dml-and-transactions) +- [111. test_objstore_credentials.py: object-store credentials live on a user mapping, and the environment is a privilege](#111-test_objstore_credentialspy-object-store-credentials-live-on-a-user-mapping-and-the-environment-is-a-privilege) ## 1. How to read a test in here @@ -6906,3 +6907,13 @@ THE WITHDRAWAL IS REACHED ONLY BY A RACE. When one session holds a key uncommitt | `test_an_aborted_inner_block_keeps_the_outer_statements_rows` | an outer `INSERT` and an outer `DELETE` whose expression calls a function with an aborted EXCEPTION block keep heap's answer. Before #1341's fix the INSERT kept 1 row of 5000. The same loss through a read inside the block is #1360 | | `test_an_insert_racing_an_uncommitted_delete_of_its_key` | an insert of a key whose delete (or key-changing update) is uncommitted waits, then fails with 23505 if the delete rolls back and succeeds if it commits: never two rows with the key. Before #1341's fix the rollback left two | | `test_an_index_built_after_a_delete_serves_an_older_snapshot` | a REPEATABLE READ snapshot from before a committed DELETE still finds the row through an index built after it (an Index Scan, asserted), and a new snapshot does not; a UNIQUE index still builds over a key whose other holder is deleted. Before #1341's fix the old snapshot's lookup found 0 | + +## 111. test_objstore_credentials.py: object-store credentials live on a user mapping, and the environment is a privilege + +The pytest twin of `test/objstore_credentials.sh` (#393 M4, ported for #1341), every check by the shell's name. Its fixture server is its own: an http.server in a thread that verifies every request's AWS Signature Version 4 from the spec with the standard library, so a green read proves the C signer and an independent implementation agree, and that records every request. A private cluster, because phase 2 restarts the postmaster with `AWS_*` in its environment; phase 1 starts it with every `AWS_*` of the test process removed. Secrets misplaced on a server, table or (for an endpoint) a mapping are refused `HV00D`; only a superuser may set `credentials_required 'false'`; each role's own mapping decides its read (a wrong secret is `28000` from the server); a mapping with half a credential is refused `28000` by its own message before any request; no secret reaches an error, `EXPLAIN VERBOSE` or `srvoptions`; and with the environment present, only a superuser or a `credentials_required 'false'` mapping may use it. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_object_store_credentials` | placement, per-user resolution, the half-credential refusals and the request count behind them, no leak, and ambient credentials as a privilege across the restart | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 02b56ef40..915ebb33b 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -656,4 +656,6 @@ guard_tests 433 # # cluster_tests 551 -> 558 for #1341's test_am_contracts.py, the twin of # test/am_contracts.sh: seven tests, by collection over the cluster files on this tree. -cluster_tests 558 +# cluster_tests 558 -> 559 for #1341's test_objstore_credentials.py, the twin of +# objstore_credentials.sh: one test, by collection over the cluster files on this tree. +cluster_tests 559 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 7860fdaca..2bc13c64c 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -124,6 +124,7 @@ "native_reclaim_cycles", "native_reclaim_frag", "native_vacuum_race", + "objstore_credentials", "parallel_am_scan", "parallel_scan_cost", "partition_lifecycle", diff --git a/test/pytest/test_objstore_credentials.py b/test/pytest/test_objstore_credentials.py new file mode 100644 index 000000000..866fa47d1 --- /dev/null +++ b/test/pytest/test_objstore_credentials.py @@ -0,0 +1,336 @@ +"""Object-store credentials: placement, per-user resolution, half a credential, no leak, +and ambient credentials as a privilege (#393 M4): the pytest twin of +`test/objstore_credentials.sh` (#1341). Every check carries the shell's name. + +Non-secret config (endpoint, region) lives on the SERVER and secrets only on a USER +MAPPING, refused HV00D anywhere else at DDL time. Ambient (postmaster environment) +credentials are a privilege: a superuser, or a mapping a superuser marked +credentials_required 'false'. + +THE FIXTURE SERVER IS THIS FILE'S OWN: an http.server in a thread that serves one +Parquet file from a bucket directory and verifies every request's AWS Signature +Version 4 itself, from the AWS spec with the standard library, so a green read proves +the C signer and an independent implementation agree on the canonical request. A +request it cannot verify gets 403. It records every request, so "refused before any +request" is measured, not inferred. + +A PRIVATE CLUSTER, because phase 2 restarts the postmaster with AWS_* in its +environment, which the shared cluster cannot do. Phase 1 starts it with every AWS_* +variable of this process REMOVED, so no ambient credential can make a phase-1 read +pass. The roles are this cluster's own. +""" +import datetime +import hashlib +import hmac +import os +import pathlib +import re +import shutil +import subprocess +import tempfile +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +AKID, SECRET, BADSECRET = "PGCTESTKEYID", "pgctest-good-secret", "pgctest-wrong-secret" +REGION, BUCKET = "pgc-test-1", "pgc-bucket" +G, NCOLS, RPG = 3, 4, 6000 +AUTH = re.compile(r"^AWS4-HMAC-SHA256 Credential=([^/]+)/(\d{8})/([^/]+)/s3/aws4_request,\s*" + r"SignedHeaders=([^,]+),\s*Signature=([0-9a-f]{64})$") +RANGE = re.compile(r"^bytes=(\d*)-(\d*)$") + + +def _uri_encode(s): + return urllib.parse.quote(s, safe="-_.~") + + +def _canonical_query(raw): + """Each name and value decoded, then strictly re-encoded, sorted by name then value.""" + if not raw: + return "" + pairs = [] + for part in raw.split("&"): + k, _, v = part.partition("=") + pairs.append((_uri_encode(urllib.parse.unquote_plus(k)), _uri_encode(urllib.parse.unquote_plus(v)))) + return "&".join(f"{k}={v}" for k, v in sorted(pairs)) + + +def _signature(secret, method, raw_path, raw_query, headers, signed, amzdate, datestamp, region, payload): + canon = "".join(f"{h}:{' '.join((headers.get(h) or '').split())}\n" for h in signed) + creq = "\n".join([method, raw_path, _canonical_query(raw_query), canon, ";".join(signed), payload]) + to_sign = "\n".join(["AWS4-HMAC-SHA256", amzdate, f"{datestamp}/{region}/s3/aws4_request", + hashlib.sha256(creq.encode()).hexdigest()]) + key = ("AWS4" + secret).encode() + for part in (datestamp, region, "s3", "aws4_request"): + key = hmac.new(key, part.encode(), hashlib.sha256).digest() + return hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest() + + +class _SigV4(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _verified(self): + self.server.requests.append((self.command, self.path)) + m = AUTH.match(self.headers.get("Authorization", "")) + ok = False + if m and m.group(1) == AKID and m.group(3) == REGION: + raw_path, _, raw_query = self.path.partition("?") + want = _signature(SECRET, self.command, raw_path, raw_query, self.headers, + m.group(4).split(";"), self.headers.get("x-amz-date", ""), m.group(2), + REGION, self.headers.get("x-amz-content-sha256", "")) + ok = hmac.compare_digest(want, m.group(5)) + if not ok: + body = b'SignatureDoesNotMatch' + self.send_response(403) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + return ok + + def _file(self): + return pathlib.Path(self.server.root) / urllib.parse.unquote(self.path.partition("?")[0]).lstrip("/") + + def do_HEAD(self): + if not self._verified(): + return + f = self._file() + if not f.is_file(): + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Length", str(f.stat().st_size)) + self.send_header("Accept-Ranges", "bytes") + self.end_headers() + + def do_GET(self): + if not self._verified(): + return + f = self._file() + if not f.is_file(): + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + return + data = f.read_bytes() + m = RANGE.match((self.headers.get("Range") or "").replace(" ", "")) + if not m: + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + if m.group(1) == "": + lo, hi = max(0, len(data) - int(m.group(2))), len(data) - 1 + else: + lo = int(m.group(1)) + hi = min(int(m.group(2)) if m.group(2) else len(data) - 1, len(data) - 1) + self.send_response(206) + self.send_header("Content-Range", f"bytes {lo}-{hi}/{len(data)}") + self.send_header("Content-Length", str(hi - lo + 1)) + self.end_headers() + self.wfile.write(data[lo:hi + 1]) + + +def _pg_ctl(cluster, args, extra_env): + """pg_ctl with an explicit environment: every AWS_* of this process removed, then + `extra_env` added. Dropped to postgres when running as root, as the harness does.""" + unset = [a for k in os.environ if k.startswith("AWS_") for a in ("-u", k)] + cmd = ["env"] + unset + [f"{k}={v}" for k, v in extra_env.items()] + \ + [os.path.join(cluster.bindir, "pg_ctl")] + args + if os.geteuid() == 0: + cmd = ["runuser", "-u", "postgres", "--"] + cmd + return subprocess.run(cmd, capture_output=True, text=True) + + +def test_object_store_credentials(pgc_cluster, expect): + pa = expect.require_module("pyarrow", name="pyarrow is required to write the fixture") + if pa is None: + return + import psycopg + # Through require_module, as the import guard requires: pyarrow.parquet is a + # submodule a bare "import pyarrow" does not load. + pq = expect.require_module("pyarrow.parquet", name="pyarrow is required to write the fixture") + if pq is None: + return + from pgc_cluster import Cluster, pick_port + + work = pathlib.Path(tempfile.mkdtemp(prefix="pgc-creds-")) + os.chmod(work, 0o777) + (work / BUCKET).mkdir() + os.chmod(work / BUCKET, 0o777) + fixture = work / BUCKET / "fixture.parquet" + n = G * RPG + pq.write_table(pa.table({f"c{i}": pa.array([(i + 1) * v for v in range(n)], pa.int64()) + for i in range(NCOLS)}), + str(fixture), row_group_size=RPG, data_page_size=4096, compression="NONE", + use_dictionary=False) + os.chmod(fixture, 0o644) + srv = ThreadingHTTPServer(("127.0.0.1", 0), _SigV4) + srv.root, srv.requests = str(work), [] + threading.Thread(target=srv.serve_forever, daemon=True).start() + port = srv.server_address[1] + import urllib.request + try: + urllib.request.urlopen(urllib.request.Request(f"http://127.0.0.1:{port}/{BUCKET}/fixture.parquet", + method="HEAD"), timeout=5) + answered = "unsigned request admitted" + except urllib.error.HTTPError as e: + answered = "403" if e.code == 403 else f"http {e.code}" + except OSError as e: + answered = f"no answer: {e}" + expect.text(answered, "403", "premise: sigv4 fixture server is up") + + datadir = work / "data" + datadir.mkdir() + cluster = Cluster(pgc_cluster.pg_config, "creds", datadir, pick_port(9)) + log = str(work / "server.log") + started = False + try: + cluster.initdb() + with open(datadir / "postgresql.conf", "a") as fh: + fh.write("\npgcolumnar.objstore_allowed_endpoints='127.0.0.1'\n") + out = _pg_ctl(cluster, ["-D", str(datadir), "-l", log, "-w", "start"], {}) + started = out.returncode == 0 + if not started: + raise RuntimeError(f"the private cluster did not start: {out.stderr}") + dsn = cluster.dsn() + + def sql(stmt, role="postgres"): + """-> (SQLSTATE or '', first value or None, primary message or '').""" + try: + with psycopg.connect(dsn.replace("user=postgres", f"user={role}"), autocommit=True) as c: + cur = c.execute(stmt) + row = cur.fetchone() if cur.description else None + return "", (row[0] if row else None), "" + except psycopg.Error as e: + return e.sqlstate or "?", None, e.diag.message_primary or "" + + def count(stmt, role="postgres"): + """-> the count, or -1 when the read is refused: a refusal is a red arm, not a crash.""" + state, value, _ = sql(stmt, role) + return -1 if state or value is None else value + + def run(*stmts): + with psycopg.connect(dsn, autocommit=True) as c: + for s in stmts: + c.execute(s) + + def table_hash(t): + return sql(f"SELECT md5(string_agg(x::text, ',' ORDER BY x::text)) FROM {t} x")[1] + + cols = ", ".join(f"c{i} int8" for i in range(NCOLS)) + run("CREATE EXTENSION IF NOT EXISTS pgcolumnar") + for r in ("cred_alice", "cred_bob", "cred_carol"): + run(f"CREATE ROLE {r} LOGIN", f"GRANT pg_read_server_files TO {r}") + + # ---- placement: the validator split + expect.text(sql("CREATE SERVER badsrv FOREIGN DATA WRAPPER pgcolumnar_parquet" + " OPTIONS (secret_access_key 'x')")[0], "HV00D", "placement: secret on a SERVER is HV00D") + run("CREATE SERVER tmpsrv FOREIGN DATA WRAPPER pgcolumnar_parquet") + expect.text(sql(f"CREATE FOREIGN TABLE ft_bad ({cols}) SERVER tmpsrv OPTIONS (endpoint 'x')")[0], + "HV00D", "placement: endpoint on a TABLE is HV00D") + run(f"CREATE SERVER csrv FOREIGN DATA WRAPPER pgcolumnar_parquet" + f" OPTIONS (endpoint 'http://127.0.0.1:{port}', region '{REGION}')") + expect.num(count("SELECT count(*) FROM pg_foreign_server WHERE srvname = 'csrv'"), 1, + "placement: endpoint and region on the SERVER are accepted") + expect.text(sql(f"CREATE USER MAPPING FOR postgres SERVER csrv" + f" OPTIONS (access_key_id '{AKID}', secret_access_key '{SECRET}')")[0] or "ok", "ok", + "placement: a credential on a USER MAPPING is accepted") + expect.text(sql("CREATE USER MAPPING FOR cred_alice SERVER csrv OPTIONS (endpoint 'x')")[0], "HV00D", + "placement: endpoint on a USER MAPPING is HV00D") + run("GRANT USAGE ON FOREIGN SERVER csrv TO cred_alice, cred_bob, cred_carol", + f"CREATE USER MAPPING FOR cred_alice SERVER csrv OPTIONS (access_key_id '{AKID}', secret_access_key '{SECRET}')", + f"CREATE USER MAPPING FOR cred_bob SERVER csrv OPTIONS (access_key_id '{AKID}', secret_access_key '{BADSECRET}')") + expect.text(sql("ALTER USER MAPPING FOR cred_alice SERVER csrv OPTIONS (ADD credentials_required 'false')", + "cred_alice")[0], "42501", + "placement: a non-superuser cannot set credentials_required false (42501)") + expect.text(sql("CREATE USER MAPPING FOR cred_carol SERVER csrv OPTIONS (credentials_required 'false')")[0] + or "ok", "ok", "placement: a superuser can (for carol, used below)") + + run(f"CREATE FOREIGN TABLE flocal ({cols}) SERVER csrv OPTIONS (path '{fixture}')", + f"CREATE FOREIGN TABLE fs3 ({cols}) SERVER csrv OPTIONS (path 's3://{BUCKET}/fixture.parquet')", + "GRANT SELECT ON flocal, fs3 TO cred_alice, cred_bob, cred_carol") + + # ---- phase 1: no AWS environment anywhere + run("DROP USER MAPPING FOR postgres SERVER csrv") + expect.text(sql("SELECT count(*) FROM fs3")[0], "28000", + "premise: without mapping or environment, even a superuser gets 28000") + run(f"CREATE USER MAPPING FOR postgres SERVER csrv OPTIONS (access_key_id '{AKID}', secret_access_key '{SECRET}')") + expect.text(table_hash("fs3") or "unread", table_hash("flocal") or "local unread", + "catalog: mapping + server options reach the wire (differential)") + expect.num(count("SELECT count(*) FROM fs3"), n, "catalog: count(*) with catalog credentials only") + expect.num(count("SELECT count(*) FROM fs3", "cred_alice"), n, + "per-user: alice (good secret in her mapping) reads") + expect.text(sql("SELECT count(*) FROM fs3", "cred_bob")[0], "28000", + "per-user: bob (wrong secret in his mapping) is refused 28000") + expect.text(sql("SELECT count(*) FROM fs3", "cred_carol")[0], "28000", + "ambient-is-privilege: carol (credentials_required=false, but no env exists) is 28000") + + # ---- half a credential (#1341): refused before any request, by its own message + for r in ("cred_dave", "cred_erin"): + run(f"CREATE ROLE {r} LOGIN", f"GRANT pg_read_server_files TO {r}", + f"GRANT USAGE ON FOREIGN SERVER csrv TO {r}", f"GRANT SELECT ON fs3 TO {r}") + run(f"CREATE USER MAPPING FOR cred_dave SERVER csrv OPTIONS (access_key_id '{AKID}')", + f"CREATE USER MAPPING FOR cred_erin SERVER csrv OPTIONS (secret_access_key '{SECRET}')") + half = '28000 columnar: the user mapping for server "csrv" carries only half a credential' + before = len(srv.requests) + sql("SELECT count(*) FROM fs3", "cred_alice") + expect.at_least(len(srv.requests) - before, 1, + "control: a full mapping's read reaches the endpoint, so the log counts requests") + before = len(srv.requests) + + def refusal(role): + state, _, msg = sql("SELECT count(*) FROM fs3", role) + return f"{state} {msg}" if state else "ok" + + expect.text(refusal("cred_dave"), half, + "a mapping with an access key and no secret is refused before any request (28000)") + expect.text(refusal("cred_erin"), half, "and one with a secret and no access key is refused the same way") + expect.num(len(srv.requests) - before, 0, "and neither half-credential read reached the endpoint") + + # ---- no leak + state, _, bob_msg = sql("SELECT count(*) FROM fs3", "cred_bob") + with psycopg.connect(dsn, autocommit=True) as c: + explain = "\n".join(r[0] for r in c.execute("EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM fs3").fetchall()) + expect.num(sum(s in bob_msg for s in (SECRET, BADSECRET)), 0, "no-leak: bob's verbose error carries no secret") + expect.text("yes" if state else "no", "yes", "premise: bob's error was a real refusal, not silence") + expect.num(sum(s in explain for s in (SECRET, BADSECRET)), 0, "no-leak: EXPLAIN VERBOSE carries no secret") + expect.num(count(f"SELECT count(*) FROM pg_foreign_server WHERE srvoptions::text LIKE '%{SECRET}%'"), 0, + "no-leak: srvoptions carry no secret (by construction)") + + # ---- a PUBLIC mapping (#1341): the user's own mapping first, then PUBLIC. Frank + # has none of his own; bob's own carries the wrong secret, which a PUBLIC mapping + # holding the right one must not override. Dropped before phase 2. + run("CREATE ROLE cred_frank LOGIN", "GRANT pg_read_server_files TO cred_frank", + "GRANT USAGE ON FOREIGN SERVER csrv TO cred_frank", "GRANT SELECT ON fs3 TO cred_frank", + f"CREATE USER MAPPING FOR PUBLIC SERVER csrv OPTIONS (access_key_id '{AKID}', secret_access_key '{SECRET}')") + expect.num(count("SELECT count(*) FROM fs3", "cred_frank"), n, + "a role with no mapping of its own reads through the PUBLIC mapping") + expect.text(sql("SELECT count(*) FROM fs3", "cred_bob")[0], "28000", + "a role's own mapping is used before the PUBLIC one (bob's wrong secret is still refused 28000)") + run("DROP USER MAPPING FOR PUBLIC SERVER csrv") + + # ---- phase 2: the environment is present, and ambient is still a privilege + _pg_ctl(cluster, ["-D", str(datadir), "-m", "fast", "-w", "stop"], {}) + started = _pg_ctl(cluster, ["-D", str(datadir), "-l", log, "-w", "start"], + {"AWS_ENDPOINT_URL": f"http://127.0.0.1:{port}", "AWS_ACCESS_KEY_ID": AKID, + "AWS_SECRET_ACCESS_KEY": SECRET, "AWS_REGION": REGION}).returncode == 0 + expect.num(count("SELECT count(*) FROM fs3", "cred_carol"), n, + "ambient: carol (credentials_required=false) now reads via the environment") + run("DROP USER MAPPING FOR cred_carol SERVER csrv") + expect.text(sql("SELECT count(*) FROM fs3", "cred_carol")[0], "28000", + "ambient: without her mapping, carol is refused despite the environment") + run("DROP USER MAPPING FOR postgres SERVER csrv") + expect.num(count("SELECT count(*) FROM fs3"), n, + "ambient: a superuser with no mapping reads via the environment") + finally: + if started: + _pg_ctl(cluster, ["-D", str(datadir), "-m", "immediate", "-w", "stop"], {}) + srv.shutdown() + shutil.rmtree(work, ignore_errors=True)