test: add loadtest with normally distributed server delay

CONTRIBUTING.md

@@ -128,7 +128,7 @@ $ sudo net-with-gdb
 
 The `net-loadtest` launchs a temporary db and a nginx server, waiting until a number of requests are done, then reporting results (plus process monitoring) at the end.
 
-It takes a two parameters: the number of requests (GETs) and the `pg_net.batch_size`.
+It takes up to four optional parameters: the number of requests (GETs), the `pg_net.batch_size`, and a server delay in ms with its standard deviation in ms (normally distributed). Run `net-loadtest --help` for details.
 
 ```bash
 $ net-loadtest 1000 200

nix/loadtest.nix

@@ -67,6 +67,31 @@ in
 writeShellScriptBin "net-loadtest" ''
   set -euo pipefail
 
+  usage() {
+    cat <<EOF
+  Usage: net-loadtest [REQUESTS] [BATCH_SIZE] [DELAY_MS] [DELAY_SD_MS]
+
+  Launch a temporary db and nginx server, wait until REQUESTS GETs are done,
+  then report results plus process monitoring (CPU/MEM) as markdown.
+
+  Arguments (all optional, positional):
+    NUM_REQUESTS  number of GET requests (default: 10000)
+    BATCH_SIZE    value for pg_net.batch_size (default: extension default)
+    DELAY_MS      mean server delay in ms, normally distributed (default: none)
+    DELAY_SD_MS   standard deviation of the server delay in ms (default: 0)
+
+  Options:
+    -h, --help   show this help and exit
+
+  Example:
+    net-loadtest 1000 200 50 10
+  EOF
+  }
+
+  case "''${1:-}" in
+    -h|--help) usage; exit 0 ;;
+  esac
+
   reqs=""
   batch_size_opt=""
 
@@ -88,6 +113,12 @@ writeShellScriptBin "net-loadtest" ''
     batch_size_opt="-c pg_net.batch_size=$2"
   fi
 
+  # Optional server delay in ms following a normal distribution.
+  # Arguments: delay in ms ($3), delay standard deviation in ms ($4)
+  if [ -n "''${3:-}" ]; then
+    reqs="$reqs, delay_ms => ''${3}, delay_sd_ms => ''${4:-0}"
+  fi
+
   net-with-nginx xpg --options "-c log_min_messages=WARNING $batch_size_opt" \
     psql -c "call wait_for_many_gets($reqs)" -c "\pset format csv" -c "\o $query_csv" -c "select * from run" > /dev/null &
 

nix/nginx/conf/custom.conf

@@ -2,14 +2,9 @@ location / {
   echo 'Hello world';
 }
 
-location /slow-reply {
-  echo_sleep 2;
-  echo 'this text will come in response body with HTTP 200 after 2 seconds';
-}
-
-location /really-slow-reply {
-  echo_sleep 30;
-  echo 'this text will come in response body with HTTP 200 after 30 seconds';
+location /delay {
+  echo_sleep $arg_s;
+  echo 'ok';
 }
 
 location /echo-method {

test/utils/loadtest.sql

@@ -7,8 +7,29 @@ create table run (
   last_failure_error text
 );
 
+-- random delay in ms from a normal distribution with the given mean and standard deviation.
+-- Uses random_normal(), which is only available on PostgreSQL 16+.
+-- Negative results are clamped to 0 since a delay can't be negative.
+create or replace function random_delay_ms(mean_ms int, sd_ms int) returns int as $$
+begin
+  if current_setting('server_version_num')::int < 160000 then
+    raise exception 'function `random_normal` requires PostgreSQL 16 or newer, running %', current_setting('server_version');
+  end if;
+  return greatest(0, round(random_normal(mean_ms, sd_ms)))::int;
+end;
+$$ language plpgsql volatile;
+
 -- loadtest using many gets, used to be called `repro_timeouts`
-create or replace procedure wait_for_many_gets(number_of_requests int default 10000, url text default 'http://localhost:8080') as $$
+--
+-- when delay_ms is given, each request goes to the nginx /delay endpoint with a server delay drawn
+-- from a normal distribution with mean delay_ms and standard deviation delay_sd_ms (clamped
+-- at 0), to simulate the latency of real servers
+create or replace procedure wait_for_many_gets(
+  number_of_requests int default 10000,
+  url text default 'http://localhost:8080',
+  delay_ms int default null,
+  delay_sd_ms int default 0
+) as $$
 declare
   last_id bigint;
   first_time timestamptz;
@@ -22,7 +43,11 @@ begin
 
   with do_requests as (
     select
-      net.http_get(url) as id
+      net.http_get(
+        case when delay_ms is null then url
+        else url || '/delay?s=' || round(random_delay_ms(delay_ms, delay_sd_ms) / 1000.0, 3)
+        end
+      ) as id
     from generate_series (1, number_of_requests) x
   )
   select id, clock_timestamp() into last_id, first_time from do_requests offset number_of_requests - 1;