Base CTID split part count on total size (heap + TOAST)

docs/concurrency.rst

@@ -234,6 +234,12 @@ partitioned (on the fly) and split between processes:
        COPY (SELECT * FROM source.table WHERE ctid >= '(17775,0)'::tid and ctid < '(23698,0)'::tid)
        COPY (SELECT * FROM source.table WHERE ctid >= '(23698,0)'::tid)
 
+    The number of CTID parts is derived from the total on-disk size of the
+    table (heap plus TOAST), so a table whose size is dominated by TOAST still
+    splits into a sensible number of concurrent COPY processes. A CTID range
+    scan detoasts each row it reads, so splitting the heap page range
+    parallelizes the TOAST reads as well.
+
 
   - To decide if a table COPY processing should be split, the command line
     option ``split-tables-larger-than`` is used, or the environment variable

src/bin/pgcopydb/schema.c

@@ -4144,26 +4144,28 @@ schema_list_partitions(PGSQL *pgsql,
 		max = table->relpages;
 
 		/*
-		 * Get the block size from the origin in the first attempt
-		 * and then memoize it.
+		 * Base the part count on total on-disk size (heap + TOAST) so
+		 * TOAST-dominant tables split. Under --estimate-table-sizes, bytes is
+		 * heap-only and this degrades to the prior heap-based behavior.
 		 */
-		static int blockSize = 0;
-		bool isBlockSizeCached = blockSize != 0;
-		if (!isBlockSizeCached && !pgsql_get_block_size(pgsql, &blockSize))
-		{
-			/* errors have already been logged */
-			return false;
-		}
-		uint64_t pagesPerPart = ceil((double) partSize / blockSize);
-
-		partsCount = ceil((double) table->relpages / (double) pagesPerPart);
+		partsCount = ceil((double) table->bytes / (double) partSize);
 
 		if (splitMaxParts > 0 && partsCount > splitMaxParts)
 		{
 			partsCount = splitMaxParts;
 		}
 
-		partsSize = ceil((double) table->relpages / partsCount);
+		/* cannot create more CTID page-ranges than there are heap pages */
+		if (table->relpages > 0 && partsCount > table->relpages)
+		{
+			partsCount = table->relpages;
+		}
+		else if (table->relpages == 0)
+		{
+			partsCount = 1;
+		}
+
+		partsSize = ceil((double) table->relpages / (double) partsCount);
 	}
 
 	/*

tests/unit/expected/4-list-table-split.out

@@ -33,17 +33,24 @@
 024-05-23 09:14:52.903 184 INFO   main.c:136                Running pgcopydb version 0.15.28.g40af8d7 from "/usr/local/bin/pgcopydb"
 2024-05-23 09:14:52.941 184 INFO   copydb.c:105              Using work dir "/tmp/unit/split"
 2024-05-23 09:14:52.944 184 INFO   cli_list.c:1297           Table public.table_ctid_candidate is 152 kB large which is larger than --split-tables-larger-than 10 kB, and does not have a unique column of type integer: splitting by CTID
-2024-05-23 09:14:52.944 184 INFO   cli_list.c:1321           Table public.table_ctid_candidate COPY will be split 8-ways
+2024-05-23 09:14:52.944 184 INFO   cli_list.c:1321           Table public.table_ctid_candidate COPY will be split 15-ways
         Part |          Min |          Max |        Count
 -------------+--------------+--------------+-------------
-         1/8 |        (0,0) |        (1,0) |            2
-         2/8 |        (2,0) |        (3,0) |            2
-         3/8 |        (4,0) |        (5,0) |            2
-         4/8 |        (6,0) |        (7,0) |            2
-         5/8 |        (8,0) |        (9,0) |            2
-         6/8 |       (10,0) |       (11,0) |            2
-         7/8 |       (12,0) |       (13,0) |            2
-         8/8 |       (14,0) |       (-1,0) |           -1
+         1/15 |        (0,0) |        (0,0) |            1
+         2/15 |        (1,0) |        (1,0) |            1
+         3/15 |        (2,0) |        (2,0) |            1
+         4/15 |        (3,0) |        (3,0) |            1
+         5/15 |        (4,0) |        (4,0) |            1
+         6/15 |        (5,0) |        (5,0) |            1
+         7/15 |        (6,0) |        (6,0) |            1
+         8/15 |        (7,0) |        (7,0) |            1
+         9/15 |        (8,0) |        (8,0) |            1
+        10/15 |        (9,0) |        (9,0) |            1
+        11/15 |       (10,0) |       (10,0) |            1
+        12/15 |       (11,0) |       (11,0) |            1
+        13/15 |       (12,0) |       (12,0) |            1
+        14/15 |       (13,0) |       (13,0) |            1
+        15/15 |       (14,0) |       (-1,0) |           -1
 2024-05-23 11:01:25.505 201 INFO   main.c:136                Running pgcopydb version 0.15.28.g40af8d7 from "/usr/local/bin/pgcopydb"
 2024-05-23 11:01:25.551 201 INFO   cli_list.c:1300           Table public.table_ctid_candidate_skip is 152 kB large which is larger than --split-tables-larger-than 10 kB, does not have a unique column of type integer, and CTID split is disabled.Same table concurrency is not enabled
 
@@ -54,4 +61,10 @@
 -------------+--------------+--------------+-------------
          1/3 |           -1 |           -1 |           -1
          2/3 |            1 |           34 |           34
-         3/3 |           35 |           -1 |           -1
+         3/3 |           35 |           -1 |           -1
+20:54:39.774 91 INFO   Table public.table_toast_heavy is 6848 kB large which is larger than --split-tables-larger-than 1024 kB, and does not have a unique column of type integer: splitting by CTID
+20:54:39.774 91 INFO   Table public.table_toast_heavy COPY will be split 2-ways
+        Part |          Min |          Max |        Count
+-------------+--------------+--------------+-------------
+         1/2 |        (0,0) |        (1,0) |            2
+         2/2 |        (2,0) |       (-1,0) |           -1

tests/unit/script/4-list-table-split.sh

@@ -57,3 +57,14 @@ pgcopydb list schema --dir ${DIR} ${OPTS} >/dev/null
 pgcopydb list table-parts --dir ${DIR} \
     --schema-name "public" --table-name "table_1" \
     --split-tables-larger-than "10 kB" --split-max-parts 3 2>&1
+
+
+# TOAST-heavy table (~6.5 MB, ~4 heap pages, no integer PK): old heap-only
+# math gives 1 part (no split); the fix splits on total size, capped to 2.
+DIR=/tmp/unit/toast-split
+pgcopydb list schema --dir ${DIR} --not-consistent \
+    --split-tables-larger-than 1MB --split-max-parts 2 >/dev/null
+
+pgcopydb list table-parts --dir ${DIR} \
+    --schema-name "public" --table-name "table_toast_heavy" \
+    --split-tables-larger-than 1MB --split-max-parts 2 2>&1

tests/unit/setup/4-list-table-split.sql

@@ -98,3 +98,17 @@ select
     *
 from
     table_ctid_candidate;
+
+-- TOAST-dominant table, no integer PK: tiny heap, large out-of-line TOAST.
+-- STORAGE EXTERNAL disables compression so the size is stable across PG
+-- versions (heap ~4 pages, pg_table_size ~6.5 MB).
+
+create table table_toast_heavy (
+    id integer,
+    payload text
+);
+alter table table_toast_heavy alter column payload set storage external;
+
+insert into table_toast_heavy (id, payload)
+select g, repeat(md5(g::text), 400)
+from generate_series(1, 500) g;