From 055d3f668e21a0919a471e51b88457fabd2a2d4c Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Thu, 1 Oct 2026 15:23:45 -0500 Subject: [PATCH 1/6] refactor: create extension from beginning of lineaage Currently in pg_net the upgrade code path is different from the create extension code path. When running tests internally we are not able to do true upgrade testing because the initial release of pg_net, 0.1 is not avaiable. This commit cherry picks 0.1 from the 0.1 tag, removes the pg_net.sql file and forces all create extension to happen by creating 0.1 and upgrading all the way until the current version. This commit also corrects a missing migration that was added pg_net.sql but not in the uprade script during 8b55e96d0d4b194dd8896b6f2a0fa8ef525aa8b3. Regarding this missing migraiton, this may need an additional PR to correct somehow. It is not hard to imagine instances in the wild that do not have this modified function that should. I think this is a better approach and will open us up for upgrade testing internal to the repo. This commit --- .github/workflows/main.yml | 23 ++- .gitignore | 1 + Makefile | 15 +- pg_net.sql | 308 ++++++++++++++++++++++++++++ scripts/dump_extension.sh | 36 ++++ shell.nix | 4 +- sql/{pg_net.sql => pg_net--0.1.sql} | 171 +++++---------- sql/pg_net--0.20.4--0.20.6.sql | 78 +++++++ sql/pg_net--0.7--0.7.1.sql | 2 + sql/pg_net--0.7.1--0.7.3.sql | 55 +++++ 10 files changed, 565 insertions(+), 128 deletions(-) create mode 100644 pg_net.sql create mode 100755 scripts/dump_extension.sh rename sql/{pg_net.sql => pg_net--0.1.sql} (61%) create mode 100644 sql/pg_net--0.20.4--0.20.6.sql diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 13edb9ef..fe9a0c5b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -112,9 +112,30 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} files: ./build-${{ matrix.pg-version }}/coverage.info + validate-sql-snapshot: + runs-on: macos-15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install Nix + uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: Use Cachix Cache + uses: cachix/cachix-action@3ba601ff5bbb07c7220846facfa2cd81eeee15a1 # v16 + with: + name: nxpg + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - name: Run snapshot + run: nix develop --command "./scripts/dump_extension.sh" + + - name: Check for diffs in pg_net.sql snapshot + run: nix-shell --run "git diff --exit-code ./pg_net.sql" + style: runs-on: ubuntu-latest - steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.gitignore b/.gitignore index 0ede66aa..f4a058c8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ nginx.pid tags net_worker.pid pg_net--*.sql +!pg_net--0.1.sql !pg_net--*--*.sql diff --git a/Makefile b/Makefile index 0d1b0e35..45919ce4 100644 --- a/Makefile +++ b/Makefile @@ -33,11 +33,11 @@ else endif EXTENSION = pg_net -EXTVERSION = 0.20.4 +EXTVERSION = 0.20.6 DATA = $(wildcard sql/*--*.sql) -EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control +EXTRA_CLEAN = $(EXTENSION).control TESTS = $(wildcard test/sql/*.sql) REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS)) @@ -58,14 +58,14 @@ SHLIB_LINK = -lcurl # Find from system headers PG_CPPFLAGS := $(CPPFLAGS) -DEXTVERSION=\"$(EXTVERSION)\" -all: sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control +all: $(EXTENSION).control -build: $(BUILD_DIR)/$(EXTENSION).$(SHARED_EXT) sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control +build: $(BUILD_DIR)/$(EXTENSION).$(SHARED_EXT) $(EXTENSION).control -$(BUILD_DIR)/.gitignore: sql/$(EXTENSION)--$(EXTVERSION).sql $(EXTENSION).control +$(BUILD_DIR)/.gitignore: $(EXTENSION).control mkdir -p $(BUILD_DIR)/extension cp $(EXTENSION).control $(BUILD_DIR)/extension - cp sql/$(EXTENSION)--$(EXTVERSION).sql $(BUILD_DIR)/extension + cp sql/*.sql $(BUILD_DIR)/extension echo "*" > $(BUILD_DIR)/.gitignore $(BUILD_DIR)/%.o: src/%.c $(HEADERS) $(BUILD_DIR)/.gitignore @@ -74,9 +74,6 @@ $(BUILD_DIR)/%.o: src/%.c $(HEADERS) $(BUILD_DIR)/.gitignore $(BUILD_DIR)/$(EXTENSION).$(SHARED_EXT): $(EXTENSION).$(SHARED_EXT) mv $? $@ -sql/$(EXTENSION)--$(EXTVERSION).sql: sql/$(EXTENSION).sql - cp $< $@ - $(EXTENSION).control: $(EXTENSION).control.in sed "s/@EXTVERSION@/$(EXTVERSION)/g" $(EXTENSION).control.in > $@ diff --git a/pg_net.sql b/pg_net.sql new file mode 100644 index 00000000..4ce6dd35 --- /dev/null +++ b/pg_net.sql @@ -0,0 +1,308 @@ +CREATE SCHEMA net; + +ALTER SCHEMA net OWNER TO postgres; + +CREATE DOMAIN net.http_method AS text CONSTRAINT http_method_check CHECK (value ~~* CAST('get' AS text) + OR value ~~* CAST('post' AS text) + OR value ~~* CAST('delete' AS text)); + +ALTER DOMAIN net.http_method OWNER TO postgres; + +CREATE TYPE net.http_response AS ( status_code integer + , headers jsonb + , body text); + +ALTER TYPE net.http_response OWNER TO postgres; + +CREATE TYPE net.request_status AS ENUM ('PENDING' + , 'SUCCESS' + , 'ERROR'); + +ALTER TYPE net.request_status OWNER TO postgres; + +CREATE TYPE net.http_response_result AS ( status net.request_status + , message text + , response net.http_response); + +ALTER TYPE net.http_response_result OWNER TO postgres; + +CREATE FUNCTION net._await_response(request_id bigint) +RETURNS boolean +LANGUAGE plpgsql +AS $$ +declare + rec net._http_response; +begin + while rec is null loop + select * + into rec + from net._http_response + where id = request_id; + + if rec is null then + perform pg_sleep(0.05); + end if; + end loop; + + return true; +end; +$$; + +ALTER FUNCTION net._await_response (request_id bigint) OWNER TO postgres; + +CREATE FUNCTION net._encode_url_with_params_array(url text + , params_array text[]) +RETURNS text +LANGUAGE c IMMUTABLE AS '$libdir/pg_net' + , '_encode_url_with_params_array'; + +ALTER FUNCTION net._encode_url_with_params_array (url text, params_array text[]) OWNER TO postgres; + +CREATE FUNCTION net._http_collect_response(request_id bigint + , async boolean = TRUE) +RETURNS net.http_response_result +LANGUAGE plpgsql +AS $$ +declare + rec net._http_response; + req_exists boolean; +begin + + if not async then + perform net._await_response(request_id); + end if; + + select * + into rec + from net._http_response + where id = request_id; + + if rec is null or rec.error_msg is not null then + + + return ( + 'ERROR', + coalesce(rec.error_msg, 'request matching request_id not found'), + null + )::net.http_response_result; + + end if; + + return ( + 'SUCCESS', + 'ok', + ( + rec.status_code, + rec.headers, + rec.content + )::net.http_response + )::net.http_response_result; +end; +$$; + +ALTER FUNCTION net._http_collect_response (request_id bigint, async boolean) OWNER TO postgres; + +CREATE FUNCTION net._urlencode_string(string varchar) +RETURNS text +LANGUAGE c IMMUTABLE AS '$libdir/pg_net' + , '_urlencode_string'; + +ALTER FUNCTION net._urlencode_string (string varchar) OWNER TO postgres; + +CREATE FUNCTION net.check_worker_is_up() +RETURNS void +LANGUAGE plpgsql +AS $$ +begin + if not exists (select pid from pg_stat_activity where backend_type ilike '%pg_net%') then + raise exception using + message = 'the pg_net background worker is not up' + , detail = 'the pg_net background worker is down due to an internal error and cannot process requests' + , hint = 'make sure that you didn''t modify any of pg_net internal tables'; + end if; +end +$$; + +ALTER FUNCTION net.check_worker_is_up () OWNER TO postgres; + +COMMENT ON FUNCTION net.check_worker_is_up () + IS 'raises an exception if the pg_net background worker is not up, otherwise it doesn''t return anything'; + +CREATE FUNCTION net.http_collect_response(request_id bigint + , async boolean = TRUE) +RETURNS net.http_response_result +LANGUAGE plpgsql +AS $$ +begin; + raise notice 'The net.http_collect_response function is deprecated.'; + select net._http_collect_response(request_id, async); +end; +$$; + +ALTER FUNCTION net.http_collect_response (request_id bigint, async boolean) OWNER TO postgres; + +CREATE FUNCTION net.http_delete(url text + , params jsonb = CAST('{}' AS jsonb) + , headers jsonb = CAST('{}' AS jsonb) + , timeout_milliseconds integer = 5000 + , body jsonb = CAST(NULL AS jsonb)) +RETURNS bigint +LANGUAGE plpgsql +AS $$ +declare + request_id bigint; + params_array text[]; +begin + select coalesce(array_agg(net._urlencode_string(key) || '=' || net._urlencode_string(value)), '{}') + into params_array + from jsonb_each_text(params); + + insert into net.http_request_queue(method, url, headers, body, timeout_milliseconds) + values ( + 'DELETE', + net._encode_url_with_params_array(url, params_array), + headers, + convert_to(body::text, 'UTF8'), + timeout_milliseconds + ) + returning id + into request_id; + + perform net.wake(); + + return request_id; +end +$$; + +ALTER FUNCTION net.http_delete (url text, params jsonb, headers jsonb, timeout_milliseconds integer, body jsonb) OWNER TO postgres; + +CREATE FUNCTION net.http_get(url text + , params jsonb = CAST('{}' AS jsonb) + , headers jsonb = CAST('{}' AS jsonb) + , timeout_milliseconds integer = 5000) +RETURNS bigint +LANGUAGE plpgsql +AS $$ +declare + request_id bigint; + params_array text[]; +begin + select coalesce(array_agg(net._urlencode_string(key) || '=' || net._urlencode_string(value)), '{}') + into params_array + from jsonb_each_text(params); + + insert into net.http_request_queue(method, url, headers, timeout_milliseconds) + values ( + 'GET', + net._encode_url_with_params_array(url, params_array), + headers, + timeout_milliseconds + ) + returning id + into request_id; + + perform net.wake(); + + return request_id; +end +$$; + +ALTER FUNCTION net.http_get (url text, params jsonb, headers jsonb, timeout_milliseconds integer) OWNER TO postgres; + +CREATE FUNCTION net.http_post(url text + , body jsonb = CAST('{}' AS jsonb) + , params jsonb = CAST('{}' AS jsonb) + , headers jsonb = CAST('{"Content-Type": "application/json"}' AS jsonb) + , timeout_milliseconds integer = 5000) +RETURNS bigint +LANGUAGE plpgsql +AS $$ +declare + request_id bigint; + params_array text[]; + content_type text; +begin + + select + header_value into content_type + from + jsonb_each_text(coalesce(headers, '{}'::jsonb)) r(header_name, header_value) + where + lower(header_name) = 'content-type' + limit + 1; + + if content_type is null then + select headers || '{"Content-Type": "application/json"}'::jsonb into headers; + end if; + + if content_type <> 'application/json' then + raise exception 'Content-Type header must be "application/json"'; + end if; + + select + coalesce(array_agg(net._urlencode_string(key) || '=' || net._urlencode_string(value)), '{}') + into + params_array + from + jsonb_each_text(params); + + insert into net.http_request_queue(method, url, headers, body, timeout_milliseconds) + values ( + 'POST', + net._encode_url_with_params_array(url, params_array), + headers, + convert_to(body::text, 'UTF8'), + timeout_milliseconds + ) + returning id + into request_id; + + perform net.wake(); + + return request_id; +end +$$; + +ALTER FUNCTION net.http_post (url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) OWNER TO postgres; + +CREATE FUNCTION net.wait_until_running() +RETURNS void +LANGUAGE c AS '$libdir/pg_net' + , 'wait_until_running'; + +ALTER FUNCTION net.wait_until_running () OWNER TO postgres; + +COMMENT ON FUNCTION net.wait_until_running () + IS 'waits until the worker is running'; + +CREATE FUNCTION net.wake() +RETURNS void +LANGUAGE c AS '$libdir/pg_net' + , 'wake'; + +ALTER FUNCTION net.wake () OWNER TO postgres; + +CREATE FUNCTION net.worker_restart() +RETURNS boolean +LANGUAGE c AS '$libdir/pg_net' + , 'worker_restart'; + +ALTER FUNCTION net.worker_restart () OWNER TO postgres; + +CREATE UNLOGGED SEQUENCE net.http_request_queue_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE net.http_request_queue_id_seq OWNER TO postgres; + +GRANT USAGE + ON SCHEMA net + TO PUBLIC; + +GRANT ALL PRIVILEGES + ON SEQUENCE net.http_request_queue_id_seq + TO PUBLIC diff --git a/scripts/dump_extension.sh b/scripts/dump_extension.sh new file mode 100755 index 00000000..fe397618 --- /dev/null +++ b/scripts/dump_extension.sh @@ -0,0 +1,36 @@ +#! /usr/bin/env bash + +xpg psql \ + --file=<(cat << 'EOF' +DO $$ +DECLARE + extension_objects text[]; + extension_object text; +BEGIN + +SELECT array_agg(pg_catalog.pg_describe_object(classid, objid, 0)) into extension_objects +FROM pg_catalog.pg_depend +INNER JOIN ( + SELECT e.oid + FROM pg_catalog.pg_extension e + WHERE e.extname OPERATOR(pg_catalog.=) 'pg_net' + COLLATE pg_catalog.default + ORDER BY 1 +) AS pg_net_oid +ON pg_net_oid.oid = refobjid +WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass + AND deptype = 'e' +ORDER BY 1; + +FOR extension_object IN select unnest(extension_objects) LOOP + BEGIN + EXECUTE 'alter extension pg_net drop ' || extension_object; + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'failed on %: %', extension_object, sqlerrm; + END; +END LOOP; + +END; +$$; +EOF) \ + --command="\! pg_dump --schema=net --user=postgres --schema-only | grep -v -- -- | grep -v '^SET .*;' | grep -v 'SELECT pg_catalog.set_config' | pgpp > pg_net.sql" diff --git a/shell.nix b/shell.nix index 024abfd4..7c3b8af6 100644 --- a/shell.nix +++ b/shell.nix @@ -1,5 +1,4 @@ -let - flakeLock = builtins.fromJSON (builtins.readFile ./flake.lock); +let flakeLock = builtins.fromJSON (builtins.readFile ./flake.lock); nixpkgsLock = flakeLock.nodes.nixpkgs.locked; xpgLock = flakeLock.nodes.xpg.locked; in @@ -56,6 +55,7 @@ pkgs.mkShell { style styleCheck pkgs.ruff + pkgs.python313Packages.pglast ]; shellHook = '' export HISTFILE=.history diff --git a/sql/pg_net.sql b/sql/pg_net--0.1.sql similarity index 61% rename from sql/pg_net.sql rename to sql/pg_net--0.1.sql index b0acd86a..b05fb8a9 100644 --- a/sql/pg_net.sql +++ b/sql/pg_net--0.1.sql @@ -2,55 +2,46 @@ create schema if not exists net; create domain net.http_method as text check ( - value ilike 'get' - or value ilike 'post' - or value ilike 'delete' + value ilike 'get' or + value ilike 'post' ); -- Store pending requests. The background worker reads from here -- API: Private -create unlogged table net.http_request_queue( - id bigserial, +create table net.http_request_queue( + id bigserial primary key, method net.http_method not null, url text not null, - headers jsonb, + headers jsonb not null, body bytea, - timeout_milliseconds int not null + -- TODO: respect this + timeout_milliseconds int not null, + created timestamptz not null default now() ); -create or replace function net.check_worker_is_up() returns void as $$ -begin - if not exists (select pid from pg_stat_activity where backend_type ilike '%pg_net%') then - raise exception using - message = 'the pg_net background worker is not up' - , detail = 'the pg_net background worker is down due to an internal error and cannot process requests' - , hint = 'make sure that you didn''t modify any of pg_net internal tables'; - end if; -end -$$ language plpgsql; -comment on function net.check_worker_is_up() is 'raises an exception if the pg_net background worker is not up, otherwise it doesn''t return anything'; +create index created_idx on net.http_request_queue (created); -- Associates a response with a request -- API: Private -create unlogged table net._http_response( - id bigint, +create table net._http_response( + id bigint primary key references net.http_request_queue(id) on delete cascade, status_code integer, content_type text, headers jsonb, content text, timed_out bool, - error_msg text, - created timestamptz not null default now() + error_msg text ); -create index on net._http_response (created); - -- Blocks until an http_request is complete -- API: Private create or replace function net._await_response( request_id bigint ) returns bool + volatile + parallel safe + strict language plpgsql as $$ declare @@ -78,33 +69,21 @@ $$; create or replace function net._urlencode_string(string varchar) -- url encoded string returns text + language 'c' immutable -as 'MODULE_PATHNAME'; + strict +as 'pg_net'; -- API: Private create or replace function net._encode_url_with_params_array(url text, params_array text[]) -- url encoded string returns text + language 'c' immutable -as 'MODULE_PATHNAME'; - -create or replace function net.worker_restart() - returns bool - language 'c' -as 'MODULE_PATHNAME'; +as 'pg_net'; -create or replace function net.wait_until_running() - returns void - language 'c' -as 'MODULE_PATHNAME'; -comment on function net.wait_until_running() is 'waits until the worker is running'; - -create or replace function net.wake() - returns void - language 'c' -as 'MODULE_PATHNAME'; -- Interface to make an async request -- API: Public @@ -116,10 +95,13 @@ create or replace function net.http_get( -- key/values to be included in request headers headers jsonb default '{}'::jsonb, -- the maximum number of milliseconds the request may take before being cancelled - timeout_milliseconds int default 5000 + timeout_milliseconds int default 1000 ) -- request_id reference returns bigint + strict + volatile + parallel safe language plpgsql as $$ declare @@ -141,8 +123,6 @@ begin returning id into request_id; - perform net.wake(); - return request_id; end $$; @@ -159,10 +139,12 @@ create or replace function net.http_post( -- key/values to be included in request headers headers jsonb default '{"Content-Type": "application/json"}'::jsonb, -- the maximum number of milliseconds the request may take before being cancelled - timeout_milliseconds int DEFAULT 5000 + timeout_milliseconds int DEFAULT 1000 ) -- request_id reference returns bigint + volatile + parallel safe language plpgsql as $$ declare @@ -192,6 +174,11 @@ begin raise exception 'Content-Type header must be "application/json"'; end if; + -- Confirm body is set since http method switches on if body exists + if body is null then + raise exception 'body must not be null'; + end if; + select coalesce(array_agg(net._urlencode_string(key) || '=' || net._urlencode_string(value)), '{}') into @@ -205,58 +192,12 @@ begin 'POST', net._encode_url_with_params_array(url, params_array), headers, - convert_to(body::text, 'UTF8'), + body::text::bytea, timeout_milliseconds ) returning id into request_id; - perform net.wake(); - - return request_id; -end -$$; - --- Interface to make an async request --- API: Public -create or replace function net.http_delete( - -- url for the request - url text, - -- key/value pairs to be url encoded and appended to the `url` - params jsonb default '{}'::jsonb, - -- key/values to be included in request headers - headers jsonb default '{}'::jsonb, - -- the maximum number of milliseconds the request may take before being cancelled - timeout_milliseconds int default 5000, - -- optional body of the request - body jsonb default NULL -) - -- request_id reference - returns bigint - language plpgsql -as $$ -declare - request_id bigint; - params_array text[]; -begin - select coalesce(array_agg(net._urlencode_string(key) || '=' || net._urlencode_string(value)), '{}') - into params_array - from jsonb_each_text(params); - - -- Add to the request queue - insert into net.http_request_queue(method, url, headers, body, timeout_milliseconds) - values ( - 'DELETE', - net._encode_url_with_params_array(url, params_array), - headers, - convert_to(body::text, 'UTF8'), - timeout_milliseconds - ) - returning id - into request_id; - - perform net.wake(); - return request_id; end $$; @@ -284,8 +225,8 @@ create type net.http_response_result as ( -- Collect respones of an http request --- API: Private -create or replace function net._http_collect_response( +-- API: Public +create or replace function net.http_collect_response( -- request_id reference request_id bigint, -- when `true`, return immediately. when `false` wait for the request to complete before returning @@ -293,6 +234,9 @@ create or replace function net._http_collect_response( ) -- http response composite wrapped in a result type returns net.http_response_result + strict + volatile + parallel safe language plpgsql as $$ declare @@ -309,15 +253,27 @@ begin from net._http_response where id = request_id; - if rec is null or rec.error_msg is not null then + if rec is null then -- The request is either still processing or the request_id provided does not exist - -- TODO: request in progress is indistinguishable from request that doesn't exist + -- Check if a request exists with the given request_id + select count(1) > 0 + into req_exists + from net.http_request_queue + where id = request_id; + + if req_exists then + return ( + 'PENDING', + 'request is pending processing', + null + )::net.http_response_result; + end if; -- No request matching request_id found return ( 'ERROR', - coalesce(rec.error_msg, 'request matching request_id not found'), + 'request matching request_id not found', null )::net.http_response_result; @@ -336,22 +292,5 @@ begin end; $$; -create or replace function net.http_collect_response( - -- request_id reference - request_id bigint, - -- when `true`, return immediately. when `false` wait for the request to complete before returning - async bool default true -) - -- http response composite wrapped in a result type - returns net.http_response_result - language plpgsql -as $$ -begin - raise notice 'The net.http_collect_response function is deprecated.'; - select net._http_collect_response(request_id, async); -end; -$$; - -grant usage on schema net to PUBLIC; -grant all on all sequences in schema net to PUBLIC; -grant all on all tables in schema net to PUBLIC; +grant all on schema net to postgres; +grant all on all tables in schema net to postgres; diff --git a/sql/pg_net--0.20.4--0.20.6.sql b/sql/pg_net--0.20.4--0.20.6.sql new file mode 100644 index 00000000..6ddf6648 --- /dev/null +++ b/sql/pg_net--0.20.4--0.20.6.sql @@ -0,0 +1,78 @@ +-- begin/end below was missed in below commit +-- 8c059af7f86ea711c4b80d660bc37cd238c3b983 +create or replace function net.http_collect_response( + -- request_id reference + request_id bigint, + -- when `true`, return immediately. when `false` wait for the request to complete before returning + async bool default true +) + -- http response composite wrapped in a result type + returns net.http_response_result + volatile + language plpgsql +as $$ +begin; + raise notice 'The net.http_collect_response function is deprecated.'; + select net._http_collect_response(request_id, async); +end; +$$; + +-- Collect respones of an http request +-- API: Private +create or replace function net._http_collect_response( + -- request_id reference + request_id bigint, + -- when `true`, return immediately. when `false` wait for the request to complete before returning + async bool default true +) + -- http response composite wrapped in a result type + returns net.http_response_result + language plpgsql +as $$ +declare + rec net._http_response; + req_exists boolean; +begin + + if not async then + perform net._await_response(request_id); + end if; + + select * + into rec + from net._http_response + where id = request_id; + + if rec is null or rec.error_msg is not null then + -- The request is either still processing or the request_id provided does not exist + + -- TODO: request in progress is indistinguishable from request that doesn't exist + + -- No request matching request_id found + return ( + 'ERROR', + coalesce(rec.error_msg, 'request matching request_id not found'), + null + )::net.http_response_result; + + end if; + + -- Return a valid, populated http_response_result + return ( + 'SUCCESS', + 'ok', + ( + rec.status_code, + rec.headers, + rec.content + )::net.http_response + )::net.http_response_result; +end; +$$; + +-- below alter tables corrects drift introduced in below commit +-- 07e63144c95748cef347e13abb5c5c3fa3b1e732 +alter table net.http_request_queue alter column id drop not null; +alter table net._http_response alter column id drop not null; + + diff --git a/sql/pg_net--0.7--0.7.1.sql b/sql/pg_net--0.7--0.7.1.sql index de8e736b..2799c526 100644 --- a/sql/pg_net--0.7--0.7.1.sql +++ b/sql/pg_net--0.7--0.7.1.sql @@ -12,6 +12,8 @@ create or replace function net.http_collect_response( language plpgsql security definer as $$ +begin; raise notice 'The net.http_collect_response function is deprecated.'; select net._http_collect_response(request_id, async); +end; $$; diff --git a/sql/pg_net--0.7.1--0.7.3.sql b/sql/pg_net--0.7.1--0.7.3.sql index c9d0cc60..246df433 100644 --- a/sql/pg_net--0.7.1--0.7.3.sql +++ b/sql/pg_net--0.7.1--0.7.3.sql @@ -9,3 +9,58 @@ begin end $$ language plpgsql; comment on function net.check_worker_is_up() is 'raises an exception if the pg_net background worker is not up, otherwise it doesn''t return anything'; + + +-- Collect respones of an http request +-- API: Private +create or replace function net._http_collect_response( + -- request_id reference + request_id bigint, + -- when `true`, return immediately. when `false` wait for the request to complete before returning + async bool default true +) + -- http response composite wrapped in a result type + returns net.http_response_result + language plpgsql +as $$ +declare + rec net._http_response; + req_exists boolean; +begin + + if not async then + perform net._await_response(request_id); + end if; + + select * + into rec + from net._http_response + where id = request_id; + + if rec is null or rec.error_msg is not null then + -- The request is either still processing or the request_id provided does not exist + + -- TODO: request in progress is indistinguishable from request that doesn't exist + + -- No request matching request_id found + return ( + 'ERROR', + coalesce(rec.error_msg, 'request matching request_id not found'), + null + )::net.http_response_result; + + end if; + + -- Return a valid, populated http_response_result + return ( + 'SUCCESS', + 'ok', + ( + rec.status_code, + rec.headers, + rec.content + )::net.http_response + )::net.http_response_result; +end; +$$; + From 55c216ab805bd0e0d9fbcc9ad4c89b4458900aaf Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Tue, 20 Oct 2026 16:35:02 -0500 Subject: [PATCH 2/6] chore: format --- shell.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shell.nix b/shell.nix index 7c3b8af6..c6050dc1 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,5 @@ -let flakeLock = builtins.fromJSON (builtins.readFile ./flake.lock); +let + flakeLock = builtins.fromJSON (builtins.readFile ./flake.lock); nixpkgsLock = flakeLock.nodes.nixpkgs.locked; xpgLock = flakeLock.nodes.xpg.locked; in From 3b82783ddd875d86b27a0a92cb7bda28218a33ed Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Wed, 21 Oct 2026 15:39:48 -0500 Subject: [PATCH 3/6] fix: reviewed issues - add test for http_collect_response - fix multiple issues in http_collect_response - fix regex --- scripts/dump_extension.sh | 2 +- sql/pg_net--0.20.4--0.20.6.sql | 4 ++-- test/test_http_get_collect.py | 23 ++++++++++++++++++----- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/scripts/dump_extension.sh b/scripts/dump_extension.sh index fe397618..a3e9705f 100755 --- a/scripts/dump_extension.sh +++ b/scripts/dump_extension.sh @@ -33,4 +33,4 @@ END LOOP; END; $$; EOF) \ - --command="\! pg_dump --schema=net --user=postgres --schema-only | grep -v -- -- | grep -v '^SET .*;' | grep -v 'SELECT pg_catalog.set_config' | pgpp > pg_net.sql" + --command="\! pg_dump --schema=net --user=postgres --schema-only | grep -v -- '^--' | grep -v '^SET .*;' | grep -v 'SELECT pg_catalog.set_config' | pgpp > pg_net.sql" diff --git a/sql/pg_net--0.20.4--0.20.6.sql b/sql/pg_net--0.20.4--0.20.6.sql index 6ddf6648..d52d070e 100644 --- a/sql/pg_net--0.20.4--0.20.6.sql +++ b/sql/pg_net--0.20.4--0.20.6.sql @@ -11,9 +11,9 @@ create or replace function net.http_collect_response( volatile language plpgsql as $$ -begin; +begin raise notice 'The net.http_collect_response function is deprecated.'; - select net._http_collect_response(request_id, async); + return net._http_collect_response(request_id, async); end; $$; diff --git a/test/test_http_get_collect.py b/test/test_http_get_collect.py index 99c1e84b..a8e4d724 100644 --- a/test/test_http_get_collect.py +++ b/test/test_http_get_collect.py @@ -55,16 +55,29 @@ def test_http_get_collect_sync_success(sess): assert response["status_code"] == 200 -def test_http_collect_response_async_does_not_exist(sess): - """Collect a non-existent response with async true""" +def test_http_collect_response_async_does_not_exist(conn): + """Collect a non-existent response from net.http_collect_response with async true""" # Collect the response, waiting as needed - response = sess.execute( - text( + response = conn.execute( + """ + select * from net.http_collect_response(1, async:=true); + """ + ).fetchone() + + assert response[0] == "ERROR" + assert "not found" in response[1] + assert response[2] is None + + +def test__http_collect_response_async_does_not_exist(conn): + """Collect a non-existent response from net._http_collect_response with async true""" + + # Collect the response, waiting as needed + response = conn.execute( """ select * from net._http_collect_response(1, async:=true); """ - ) ).fetchone() assert response[0] == "ERROR" From d2c948ab6657f8ad9ce20b01dd303045c58f466b Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Wed, 21 Oct 2026 15:44:06 -0500 Subject: [PATCH 4/6] refactor: remove regex entirely --- scripts/dump_extension.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dump_extension.sh b/scripts/dump_extension.sh index a3e9705f..f55444a6 100755 --- a/scripts/dump_extension.sh +++ b/scripts/dump_extension.sh @@ -33,4 +33,4 @@ END LOOP; END; $$; EOF) \ - --command="\! pg_dump --schema=net --user=postgres --schema-only | grep -v -- '^--' | grep -v '^SET .*;' | grep -v 'SELECT pg_catalog.set_config' | pgpp > pg_net.sql" + --command="\! pg_dump --schema=net --user=postgres --schema-only | grep -v '^SET .*;' | grep -v 'SELECT pg_catalog.set_config' | pgpp > pg_net.sql" From b074a5ab758d51a6b81bcd98ace8eaa46ff3363c Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Wed, 21 Oct 2026 15:44:49 -0500 Subject: [PATCH 5/6] chore: run script --- pg_net.sql | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pg_net.sql b/pg_net.sql index 4ce6dd35..295f3b3f 100644 --- a/pg_net.sql +++ b/pg_net.sql @@ -40,6 +40,7 @@ begin where id = request_id; if rec is null then + -- Wait 50 ms before checking again perform pg_sleep(0.05); end if; end loop; @@ -78,8 +79,11 @@ begin where id = request_id; if rec is null or rec.error_msg is not null then + -- The request is either still processing or the request_id provided does not exist + -- TODO: request in progress is indistinguishable from request that doesn't exist + -- No request matching request_id found return ( 'ERROR', coalesce(rec.error_msg, 'request matching request_id not found'), @@ -88,6 +92,7 @@ begin end if; + -- Return a valid, populated http_response_result return ( 'SUCCESS', 'ok', @@ -133,9 +138,9 @@ CREATE FUNCTION net.http_collect_response(request_id bigint RETURNS net.http_response_result LANGUAGE plpgsql AS $$ -begin; +begin raise notice 'The net.http_collect_response function is deprecated.'; - select net._http_collect_response(request_id, async); + return net._http_collect_response(request_id, async); end; $$; @@ -157,6 +162,7 @@ begin into params_array from jsonb_each_text(params); + -- Add to the request queue insert into net.http_request_queue(method, url, headers, body, timeout_milliseconds) values ( 'DELETE', @@ -191,6 +197,7 @@ begin into params_array from jsonb_each_text(params); + -- Add to the request queue insert into net.http_request_queue(method, url, headers, timeout_milliseconds) values ( 'GET', @@ -223,6 +230,7 @@ declare content_type text; begin + -- Exctract the content_type from headers select header_value into content_type from @@ -232,10 +240,13 @@ begin limit 1; + -- If the user provided new headers and omitted the content type + -- add it back in automatically if content_type is null then select headers || '{"Content-Type": "application/json"}'::jsonb into headers; end if; + -- Confirm that the content-type is set as "application/json" if content_type <> 'application/json' then raise exception 'Content-Type header must be "application/json"'; end if; @@ -247,6 +258,7 @@ begin from jsonb_each_text(params); + -- Add to the request queue insert into net.http_request_queue(method, url, headers, body, timeout_milliseconds) values ( 'POST', From 5db137ba1cbc104660cb343e05bd38d4b6f9a999 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Wed, 21 Oct 2026 15:53:09 -0500 Subject: [PATCH 6/6] chore: style --- test/test_engine.py | 2 +- test/test_http_get_collect.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_engine.py b/test/test_engine.py index af8f1450..349e19dc 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -2,4 +2,4 @@ def test_connect(conn): """Sanity test verifying connection to postgres works""" conn.execute("select 1") - assert [(1, )] == conn.execute("select 1").fetchall() + assert [(1,)] == conn.execute("select 1").fetchall() diff --git a/test/test_http_get_collect.py b/test/test_http_get_collect.py index a8e4d724..591b9ddc 100644 --- a/test/test_http_get_collect.py +++ b/test/test_http_get_collect.py @@ -60,7 +60,7 @@ def test_http_collect_response_async_does_not_exist(conn): # Collect the response, waiting as needed response = conn.execute( - """ + """ select * from net.http_collect_response(1, async:=true); """ ).fetchone() @@ -75,7 +75,7 @@ def test__http_collect_response_async_does_not_exist(conn): # Collect the response, waiting as needed response = conn.execute( - """ + """ select * from net._http_collect_response(1, async:=true); """ ).fetchone()