Skip to content

Preserve PostgreSQL substring offset semantics #332

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fallintoplace wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Preserve PostgreSQL substring offset semantics #332

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/custom_types.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,14 +489,16 @@ lookup_builtin_func(Oid funcid, builtin_func_def* def) {
case F_SUBSTR_TEXT_INT4:
case F_SUBSTRING_TEXT_INT4_INT4:
case F_SUBSTRING_TEXT_INT4:
def->ch_name = "substringUTF8";
def->cf_type = CF_SUBSTRING;
def->ch_name = "\1";
return true;
/* PG substring(text, ...) counts code points */
case F_SUBSTR_BYTEA_INT4_INT4:
case F_SUBSTR_BYTEA_INT4:
case F_SUBSTRING_BYTEA_INT4_INT4:
case F_SUBSTRING_BYTEA_INT4:
def->ch_name = "substring";
def->cf_type = CF_SUBSTRING;
def->ch_name = "\1";
return true;
/* bytea variant is byte-based; CH substring matches */
case F_REGEXP_LIKE_TEXT_TEXT:
Expand Down
80 changes: 80 additions & 0 deletions src/deparse.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,8 @@ static void
deparseArrayExpr(ArrayExpr* node, deparse_expr_cxt* context);
static void
deparseArrayList(ArrayExpr* node, deparse_expr_cxt* context);
static char*
deparseExprToString(Expr* expr, deparse_expr_cxt* context);
static void
printRemoteParam(
int paramindex,
Expand Down Expand Up @@ -4718,6 +4720,84 @@ deparseFuncExpr(FuncExpr* node, deparse_expr_cxt* context) {
pfree(format);
return;
}
case CF_SUBSTRING: {
Expr* value = (Expr*)linitial(node->args);
Expr* start = (Expr*)lsecond(node->args);
bool safe_bounds;
const char* function_name =
node->funcresulttype == BYTEAOID ? "substring" : "substringUTF8";
char* valuesql = deparseExprToString(value, context);
char* startsql = deparseExprToString(start, context);
char* adjusted_start_sql;

Assert(list_length(node->args) == 2 || list_length(node->args) == 3);

safe_bounds = IsA(start, Const) && !((Const*)start)->constisnull &&
DatumGetInt32(((Const*)start)->constvalue) >= 1;
if (safe_bounds && list_length(node->args) == 3) {
Expr* length = (Expr*)lthird(node->args);

safe_bounds = IsA(length, Const) && !((Const*)length)->constisnull &&
DatumGetInt32(((Const*)length)->constvalue) >= 0;
}

if (safe_bounds) {
appendStringInfo(buf, "%s(%s, %s", function_name, valuesql, startsql);
if (list_length(node->args) == 3) {
appendStringInfoString(buf, ", ");
deparseExpr((Expr*)lthird(node->args), context);
}
appendStringInfoChar(buf, ')');

pfree(startsql);
pfree(valuesql);
return;
}

/* ClickHouse treats negative starts as offsets from the end. */
adjusted_start_sql = psprintf("if(%1$s < 1, 1, %1$s)", startsql);

if (list_length(node->args) == 2) {
appendStringInfo(
buf, "%s(%s, %s)", function_name, valuesql, adjusted_start_sql
);
} else {
Expr* length = (Expr*)lthird(node->args);
char* lengthsql = deparseExprToString(length, context);
char* adjusted_length_sql = psprintf(
"if(toInt64(%2$s) + toInt64(%1$s) > 2147483647, 2147483647, "
"if(toInt64(%2$s) + toInt64(%1$s) < 1, 0, "
"toInt64(%2$s) + toInt64(%1$s) - %3$s))",
lengthsql,
startsql,
adjusted_start_sql
);

/* PostgreSQL's substring functions are strict. */
appendStringInfo(
buf,
"if(throwIf(isNotNull(%s) AND isNotNull(%s) AND isNotNull(%s) "
"AND %s < 0, 'negative substring length not allowed'), NULL, "
"%s(%s, %s, %s))",
valuesql,
startsql,
lengthsql,
lengthsql,
function_name,
valuesql,
adjusted_start_sql,
adjusted_length_sql
);

pfree(adjusted_length_sql);
pfree(lengthsql);
}

pfree(adjusted_start_sql);
pfree(startsql);
pfree(valuesql);
return;
}
default:
break;
}
Expand Down
1 change: 1 addition & 0 deletions src/include/fdw.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,7 @@ typedef enum {
CF_PARAM_LIST_AGG, /* ordered set agg with array prams → parametric
* list aggregate function */
CF_ENCODE, /* encode(bytea, fmt) → hex/base64 family */
CF_SUBSTRING, /* substring/substr with PostgreSQL-compatible bounds */
} custom_object_type;

typedef enum {
Expand Down
80 changes: 80 additions & 0 deletions test/expected/functions.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2700,6 +2700,86 @@ SELECT val FROM t4 WHERE substr(val::bytea, 2) = 'al1'::bytea;
val1
(1 row)

-- Nonpositive or dynamic substring bounds push down with PostgreSQL-compatible
-- normalization. PostgreSQL clamps the start to 1, while ClickHouse counts
-- negative starts from the end.
EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE substring(val FROM -1) = 'val1';
QUERY PLAN
--------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((substringUTF8(val, if((-1) < 1, 1, (-1))) = 'val1'))
(3 rows)

SELECT val FROM t4 WHERE substring(val FROM -1) = 'val1';
val
------
val1
(1 row)

EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE substring(val FROM 0 FOR 2) = 'v';
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((if(throwIf(isNotNull(val) AND isNotNull(0) AND isNotNull(2) AND 2 < 0, 'negative substring length not allowed'), NULL, substringUTF8(val, if(0 < 1, 1, 0), if(toInt64(0) + toInt64(2) > 2147483647, 2147483647, if(toInt64(0) + toInt64(2) < 1, 0, toInt64(0) + toInt64(2) - if(0 < 1, 1, 0))))) = 'v'))
(3 rows)

SELECT val FROM t4 WHERE substring(val FROM 0 FOR 2) = 'v';
val
------
val1
val2
(2 rows)

EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE substring(val::bytea FROM -1 FOR 3) = 'v'::bytea;
QUERY PLAN
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((if(throwIf(isNotNull(CAST(val AS bytea(0))) AND isNotNull((-1)) AND isNotNull(3) AND 3 < 0, 'negative substring length not allowed'), NULL, substring(CAST(val AS bytea(0)), if((-1) < 1, 1, (-1)), if(toInt64((-1)) + toInt64(3) > 2147483647, 2147483647, if(toInt64((-1)) + toInt64(3) < 1, 0, toInt64((-1)) + toInt64(3) - if((-1) < 1, 1, (-1)))))) = 'v'))
(3 rows)

SELECT val FROM t4 WHERE substring(val::bytea FROM -1 FOR 3) = 'v'::bytea;
val
------
val1
val2
(2 rows)

-- PostgreSQL errors for negative lengths; preserve that behavior remotely.
EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE substring(val FROM 2 FOR -1) = 'never';
QUERY PLAN
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((if(throwIf(isNotNull(val) AND isNotNull(2) AND isNotNull((-1)) AND (-1) < 0, 'negative substring length not allowed'), NULL, substringUTF8(val, if(2 < 1, 1, 2), if(toInt64(2) + toInt64((-1)) > 2147483647, 2147483647, if(toInt64(2) + toInt64((-1)) < 1, 0, toInt64(2) + toInt64((-1)) - if(2 < 1, 1, 2))))) = 'never'))
(3 rows)

-- A parameterized offset also uses the remote normalization.
SET plan_cache_mode = force_generic_plan;
PREPARE substring_offset(int) AS
SELECT val FROM t4 WHERE substring(val FROM $1) = 'al1';
EXPLAIN (VERBOSE, COSTS OFF) EXECUTE substring_offset(2);
QUERY PLAN
-------------------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((substringUTF8(val, if({p1:Int32} < 1, 1, {p1:Int32})) = 'al1'))
(3 rows)

EXECUTE substring_offset(2);
val
------
val1
(1 row)

DEALLOCATE substring_offset;
RESET plan_cache_mode;
-- length(text) pushes down as lengthUTF8 (counts code points).
EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE length(val) = 3;
Expand Down
80 changes: 80 additions & 0 deletions test/expected/functions_0.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2700,6 +2700,86 @@ SELECT val FROM t4 WHERE substr(val::bytea, 2) = 'al1'::bytea;
val1
(1 row)

-- Nonpositive or dynamic substring bounds push down with PostgreSQL-compatible
-- normalization. PostgreSQL clamps the start to 1, while ClickHouse counts
-- negative starts from the end.
EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE substring(val FROM -1) = 'val1';
QUERY PLAN
--------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((substringUTF8(val, if((-1) < 1, 1, (-1))) = 'val1'))
(3 rows)

SELECT val FROM t4 WHERE substring(val FROM -1) = 'val1';
val
------
val1
(1 row)

EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE substring(val FROM 0 FOR 2) = 'v';
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((if(throwIf(isNotNull(val) AND isNotNull(0) AND isNotNull(2) AND 2 < 0, 'negative substring length not allowed'), NULL, substringUTF8(val, if(0 < 1, 1, 0), if(toInt64(0) + toInt64(2) > 2147483647, 2147483647, if(toInt64(0) + toInt64(2) < 1, 0, toInt64(0) + toInt64(2) - if(0 < 1, 1, 0))))) = 'v'))
(3 rows)

SELECT val FROM t4 WHERE substring(val FROM 0 FOR 2) = 'v';
val
------
val1
val2
(2 rows)

EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE substring(val::bytea FROM -1 FOR 3) = 'v'::bytea;
QUERY PLAN
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((if(throwIf(isNotNull(CAST(val AS bytea(0))) AND isNotNull((-1)) AND isNotNull(3) AND 3 < 0, 'negative substring length not allowed'), NULL, substring(CAST(val AS bytea(0)), if((-1) < 1, 1, (-1)), if(toInt64((-1)) + toInt64(3) > 2147483647, 2147483647, if(toInt64((-1)) + toInt64(3) < 1, 0, toInt64((-1)) + toInt64(3) - if((-1) < 1, 1, (-1)))))) = 'v'))
(3 rows)

SELECT val FROM t4 WHERE substring(val::bytea FROM -1 FOR 3) = 'v'::bytea;
val
------
val1
val2
(2 rows)

-- PostgreSQL errors for negative lengths; preserve that behavior remotely.
EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE substring(val FROM 2 FOR -1) = 'never';
QUERY PLAN
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((if(throwIf(isNotNull(val) AND isNotNull(2) AND isNotNull((-1)) AND (-1) < 0, 'negative substring length not allowed'), NULL, substringUTF8(val, if(2 < 1, 1, 2), if(toInt64(2) + toInt64((-1)) > 2147483647, 2147483647, if(toInt64(2) + toInt64((-1)) < 1, 0, toInt64(2) + toInt64((-1)) - if(2 < 1, 1, 2))))) = 'never'))
(3 rows)

-- A parameterized offset also uses the remote normalization.
SET plan_cache_mode = force_generic_plan;
PREPARE substring_offset(int) AS
SELECT val FROM t4 WHERE substring(val FROM $1) = 'al1';
EXPLAIN (VERBOSE, COSTS OFF) EXECUTE substring_offset(2);
QUERY PLAN
-------------------------------------------------------------------------------------------------------------------------
Foreign Scan on public.t4
Output: val
Remote SQL: SELECT val FROM functions_test.t4 WHERE ((substringUTF8(val, if({p1:Int32} < 1, 1, {p1:Int32})) = 'al1'))
(3 rows)

EXECUTE substring_offset(2);
val
------
val1
(1 row)

DEALLOCATE substring_offset;
RESET plan_cache_mode;
-- length(text) pushes down as lengthUTF8 (counts code points).
EXPLAIN (VERBOSE, COSTS OFF)
SELECT val FROM t4 WHERE length(val) = 3;
Expand Down
Loading