From df0ee09e8f21e1c661877de18e4eb92560634ae5 Mon Sep 17 00:00:00 2001 From: Kostia R Date: Fri, 24 Jul 2026 06:26:30 +0000 Subject: [PATCH 1/4] Push down JSONB existence predicates --- doc/pg_clickhouse.md | 1 + src/custom_types.c | 3 + src/deparse.c | 158 +++++++++++++++++++++++++++++++++ src/include/fdw.h | 1 + test/expected/jsonb_exists.out | 65 ++++++++++++++ test/sql/jsonb_exists.sql | 53 +++++++++++ 6 files changed, 281 insertions(+) create mode 100644 test/expected/jsonb_exists.out create mode 100644 test/sql/jsonb_exists.sql diff --git a/doc/pg_clickhouse.md b/doc/pg_clickhouse.md index d49766c2..e4902caf 100644 --- a/doc/pg_clickhouse.md +++ b/doc/pg_clickhouse.md @@ -1347,6 +1347,7 @@ equivalents as follows: * `!~*` (case insensitive regexp not match): [match](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match) * `->>` (JSON/JSONB extract element as text): [sub-column syntax](https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns) * `->` (JSON/JSONB extract): [toJSONString](https://clickhouse.com/docs/sql-reference/functions/json-functions#toJSONString) + [sub-column syntax](https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns) +* `?` (JSONB top-level key or array string existence): [JSONHas](https://clickhouse.com/docs/sql-reference/functions/json-functions#jsonhas) for objects and [JSONExtractArrayRaw](https://clickhouse.com/docs/sql-reference/functions/json-functions#jsonextractarrayraw) for arrays. This also supports compatibility views that parse a JSON document stored in a ClickHouse `String`. ### IN and NULL Semantics diff --git a/src/custom_types.c b/src/custom_types.c index ea6670dc..280ee691 100644 --- a/src/custom_types.c +++ b/src/custom_types.c @@ -951,6 +951,7 @@ chfdw_check_for_custom_type(Oid typeoid) { #define OID_TEXT_REGEX_NE_OP 642 #define OID_TEXT_IREGEX_NE_OP 1229 #define OID_JSONB_FETCHVAL_OP 3211 +#define OID_JSONB_EXISTS_OP 3247 #define OID_JSONB_FETCHVAL_TEXT_OP 3477 #define OID_JSON_FETCHVAL_OP 3962 #define OID_JSON_FETCHVAL_TEXT_OP 3963 @@ -982,6 +983,8 @@ classify_builtin_operator(Oid opoid) { case OID_JSONB_FETCHVAL_TEXT_OP: case OID_JSON_FETCHVAL_TEXT_OP: return CF_JSON_FETCHVAL_TEXT; + case OID_JSONB_EXISTS_OP: + return CF_JSON_EXISTS; case OID_ARRAY_CONTAINS_OP: return CF_ARRAY_CONTAINS; case OID_ARRAY_CONTAINED_OP: diff --git a/src/deparse.c b/src/deparse.c index 8fadef8f..24afc741 100644 --- a/src/deparse.c +++ b/src/deparse.c @@ -198,6 +198,12 @@ typedef enum ExprTruthCtx { EXPR_CTX_EXACT, } ExprTruthCtx; +typedef enum JsonbDocumentKind { + JSONB_DOCUMENT_UNSUPPORTED, + JSONB_DOCUMENT_NATIVE, + JSONB_DOCUMENT_STRING, +} JsonbDocumentKind; + /* * Functions to determine whether an expression can be evaluated safely on * remote server. @@ -258,6 +264,13 @@ deparseSQLValueFunction(SQLValueFunction* node, deparse_expr_cxt* context); static void deparseOpExpr(OpExpr* node, deparse_expr_cxt* context); static void +deparseJsonbExists( + Expr* document, + Expr* key, + JsonbDocumentKind document_kind, + deparse_expr_cxt* context +); +static void deparseOperatorName(StringInfo buf, Form_pg_operator opform); static void deparseDistinctExpr(DistinctExpr* node, deparse_expr_cxt* context); @@ -382,6 +395,8 @@ get_relation_column_alias_ids( int* relno, int* colno ); +static JsonbDocumentKind +classifyJsonbDocument(Expr* expr, Expr** document); /* * Examine each qual clause in input_conds, and classify them into two groups, @@ -894,6 +909,55 @@ classify_notin_subplan(SubPlan* subplan, PlannerInfo* root, Relids relids) { return NOTIN_SHIP_NONE; } +/* + * Recognize either a native ClickHouse JSON column imported as jsonb or the + * compatibility-view shape used for a String containing a JSON document: + * + * jsonb_in(foreign_text_column::cstring) + * + * Arbitrary jsonb expressions stay local because their remote representation + * is not known. + */ +static JsonbDocumentKind +classifyJsonbDocument(Expr* expr, Expr** document) { + while (IsA(expr, RelabelType)) { + expr = ((RelabelType*)expr)->arg; + } + + if (IsA(expr, Var) && exprType((Node*)expr) == JSONBOID) { + *document = expr; + return JSONB_DOCUMENT_NATIVE; + } + + if (IsA(expr, FuncExpr)) { + FuncExpr* func = (FuncExpr*)expr; + + if (func->funcid == F_JSONB_IN && list_length(func->args) == 1) { + Expr* input = (Expr*)linitial(func->args); + + while (IsA(input, RelabelType)) { + input = ((RelabelType*)input)->arg; + } + if (IsA(input, CoerceViaIO)) { + CoerceViaIO* cast = (CoerceViaIO*)input; + Expr* raw = cast->arg; + + while (IsA(raw, RelabelType)) { + raw = ((RelabelType*)raw)->arg; + } + if (cast->resulttype == CSTRINGOID && + exprType((Node*)raw) == TEXTOID) { + *document = raw; + return JSONB_DOCUMENT_STRING; + } + } + } + } + + *document = NULL; + return JSONB_DOCUMENT_UNSUPPORTED; +} + /* * Check if expression is safe to execute remotely, and return true if so. * @@ -1076,6 +1140,7 @@ foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx) { case T_DistinctExpr: /* struct-equivalent to OpExpr */ { OpExpr* oe = (OpExpr*)node; + CustomObjectDef* cdef; /* * Similarly, only shippable operators can be sent to remote. @@ -1086,6 +1151,24 @@ foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx) { return false; } + cdef = chfdw_check_for_custom_operator(oe->opno, NULL); + if (cdef && cdef->cf_type == CF_JSON_EXISTS) { + Expr* document; + JsonbDocumentKind document_kind = + classifyJsonbDocument((Expr*)linitial(oe->args), &document); + + if (document_kind == JSONB_DOCUMENT_UNSUPPORTED || + !foreign_expr_walker( + (Node*)document, glob_cxt, EXPR_CTX_EXACT + ) || + !foreign_expr_walker( + (Node*)lsecond(oe->args), glob_cxt, EXPR_CTX_EXACT + )) { + return false; + } + break; + } + /* * Recurse to input subexpressions. */ @@ -4930,6 +5013,70 @@ findFunction(Oid typoid, char* name) { return result; } +static void +deparseJsonbDocument( + Expr* document, + JsonbDocumentKind document_kind, + deparse_expr_cxt* context +) { + if (document_kind == JSONB_DOCUMENT_NATIVE) { + appendStringInfoString(context->buf, "toJSONString("); + } + deparseExpr(document, context); + if (document_kind == JSONB_DOCUMENT_NATIVE) { + appendStringInfoChar(context->buf, ')'); + } +} + +static void +deparseJsonbExists( + Expr* document, + Expr* key, + JsonbDocumentKind document_kind, + deparse_expr_cxt* context +) { + StringInfo buf = context->buf; + + appendStringInfoString(buf, "(if(isNull("); + deparseJsonbDocument(document, document_kind, context); + appendStringInfoString(buf, ") OR isNull("); + deparseExpr(key, context); + appendStringInfoString(buf, "), NULL, "); + + if (document_kind == JSONB_DOCUMENT_STRING) { + appendStringInfoString(buf, "if(NOT isValidJSON("); + deparseJsonbDocument(document, document_kind, context); + appendStringInfoString( + buf, + "), throwIf(1, 'invalid input syntax for type json'), " + ); + } + + appendStringInfoString(buf, "multiIf(JSONType("); + deparseJsonbDocument(document, document_kind, context); + appendStringInfoString(buf, ") = 'Object', JSONHas("); + deparseJsonbDocument(document, document_kind, context); + appendStringInfoString(buf, ", "); + deparseExpr(key, context); + appendStringInfoString(buf, "), JSONType("); + deparseJsonbDocument(document, document_kind, context); + appendStringInfoString( + buf, + ") = 'Array', arrayExists(jsonb_exists_element -> " + "JSONType(jsonb_exists_element) = 'String' AND " + "JSONExtractString(jsonb_exists_element) = " + ); + deparseExpr(key, context); + appendStringInfoString(buf, ", JSONExtractArrayRaw("); + deparseJsonbDocument(document, document_kind, context); + appendStringInfoString(buf, ")), 0)"); + + if (document_kind == JSONB_DOCUMENT_STRING) { + appendStringInfoChar(buf, ')'); + } + appendStringInfoString(buf, "))"); +} + /* * Deparse given operator expression. To avoid problems around priority of * operations, we always parenthesize the arguments. @@ -5060,6 +5207,17 @@ deparseOpExpr(OpExpr* node, deparse_expr_cxt* context) { appendStringInfoChar(buf, ')'); goto cleanup; } break; + case CF_JSON_EXISTS: { + Expr* document; + JsonbDocumentKind document_kind = + classifyJsonbDocument(linitial(node->args), &document); + + Assert(document_kind != JSONB_DOCUMENT_UNSUPPORTED); + deparseJsonbExists( + document, lsecond(node->args), document_kind, context + ); + goto cleanup; + } break; case CF_JSON_FETCHVAL: case CF_JSON_FETCHVAL_TEXT: { Expr* arg1 = linitial(node->args); diff --git a/src/include/fdw.h b/src/include/fdw.h index e5e4669a..8ad5843f 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -388,6 +388,7 @@ typedef enum { CF_REGEX_ICASE_NO_MATCH, /* !~* case-insensitive regex operator */ CF_JSON_FETCHVAL, /* -> operator on json & jsonb */ CF_JSON_FETCHVAL_TEXT, /* ->> operator on json & jsonb */ + CF_JSON_EXISTS, /* ? operator on jsonb */ CF_JSON_EXTRACT_PATH_TEXT, /* jsonb?_extract_path_text() → * col."k1"."k2" */ CF_JSON_EXTRACT_PATH, /* json?_extract_path() → diff --git a/test/expected/jsonb_exists.out b/test/expected/jsonb_exists.out new file mode 100644 index 00000000..741e31b6 --- /dev/null +++ b/test/expected/jsonb_exists.out @@ -0,0 +1,65 @@ +CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw + OPTIONS (dbname 'jsonb_exists', driver 'binary'); +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +CREATE FOREIGN TABLE jsonb_exists_native ( + id integer, + document jsonb +) SERVER jsonb_exists_binary OPTIONS (table_name 'native_documents'); +CREATE FOREIGN TABLE jsonb_exists_string ( + id integer, + document text +) SERVER jsonb_exists_binary OPTIONS (table_name 'string_documents'); +CREATE VIEW jsonb_exists_compatibility AS +SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document +FROM jsonb_exists_string; +CREATE FUNCTION jsonb_exists_remote_sql(query text) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + plan jsonb; +BEGIN + EXECUTE 'EXPLAIN (FORMAT JSON, VERBOSE, COSTS OFF) ' || query INTO plan; + RETURN plan #>> '{0,Plan,Remote SQL}'; +END +$$; +SELECT jsonb_exists_remote_sql( + $$SELECT id FROM jsonb_exists_native WHERE document ? 'key'$$ +) LIKE '%JSONHas%'; + ?column? +---------- + t +(1 row) + +SELECT jsonb_exists_remote_sql( + $$SELECT id FROM jsonb_exists_compatibility WHERE document ? 'key'$$ +) LIKE '%isValidJSON%'; + ?column? +---------- + t +(1 row) + +SELECT jsonb_exists_remote_sql( + $$SELECT id FROM jsonb_exists_compatibility + WHERE document ? 'first' OR document ? 'second'$$ +) LIKE '%JSONHas%JSONHas%'; + ?column? +---------- + t +(1 row) + +SELECT jsonb_exists_remote_sql( + $$SELECT id FROM jsonb_exists_compatibility + WHERE (document || '{}'::jsonb) ? 'key'$$ +) NOT LIKE '%JSONHas%'; + ?column? +---------- + t +(1 row) + +DROP VIEW jsonb_exists_compatibility; +DROP FUNCTION jsonb_exists_remote_sql(text); +DROP FOREIGN TABLE jsonb_exists_string; +DROP FOREIGN TABLE jsonb_exists_native; +DROP SERVER jsonb_exists_binary CASCADE; +NOTICE: drop cascades to user mapping for ubuntu on server jsonb_exists_binary diff --git a/test/sql/jsonb_exists.sql b/test/sql/jsonb_exists.sql new file mode 100644 index 00000000..a4280ba0 --- /dev/null +++ b/test/sql/jsonb_exists.sql @@ -0,0 +1,53 @@ +CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw + OPTIONS (dbname 'jsonb_exists', driver 'binary'); +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; + +CREATE FOREIGN TABLE jsonb_exists_native ( + id integer, + document jsonb +) SERVER jsonb_exists_binary OPTIONS (table_name 'native_documents'); + +CREATE FOREIGN TABLE jsonb_exists_string ( + id integer, + document text +) SERVER jsonb_exists_binary OPTIONS (table_name 'string_documents'); + +CREATE VIEW jsonb_exists_compatibility AS +SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document +FROM jsonb_exists_string; + +CREATE FUNCTION jsonb_exists_remote_sql(query text) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + plan jsonb; +BEGIN + EXECUTE 'EXPLAIN (FORMAT JSON, VERBOSE, COSTS OFF) ' || query INTO plan; + RETURN plan #>> '{0,Plan,Remote SQL}'; +END +$$; + +SELECT jsonb_exists_remote_sql( + $$SELECT id FROM jsonb_exists_native WHERE document ? 'key'$$ +) LIKE '%JSONHas%'; + +SELECT jsonb_exists_remote_sql( + $$SELECT id FROM jsonb_exists_compatibility WHERE document ? 'key'$$ +) LIKE '%isValidJSON%'; + +SELECT jsonb_exists_remote_sql( + $$SELECT id FROM jsonb_exists_compatibility + WHERE document ? 'first' OR document ? 'second'$$ +) LIKE '%JSONHas%JSONHas%'; + +SELECT jsonb_exists_remote_sql( + $$SELECT id FROM jsonb_exists_compatibility + WHERE (document || '{}'::jsonb) ? 'key'$$ +) NOT LIKE '%JSONHas%'; + +DROP VIEW jsonb_exists_compatibility; +DROP FUNCTION jsonb_exists_remote_sql(text); +DROP FOREIGN TABLE jsonb_exists_string; +DROP FOREIGN TABLE jsonb_exists_native; +DROP SERVER jsonb_exists_binary CASCADE; From ad8e05c62a06b210706551e0b7fd6f64b2addc86 Mon Sep 17 00:00:00 2001 From: Kostia R Date: Fri, 24 Jul 2026 06:49:57 +0000 Subject: [PATCH 2/4] Handle nullable JSONB existence inputs --- src/deparse.c | 21 ++++++++++++++++----- test/expected/jsonb_exists.out | 2 +- test/sql/jsonb_exists.sql | 2 +- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/deparse.c b/src/deparse.c index 24afc741..c3d49995 100644 --- a/src/deparse.c +++ b/src/deparse.c @@ -5028,6 +5028,17 @@ deparseJsonbDocument( } } +static void +deparseJsonbNonnullDocument( + Expr* document, + JsonbDocumentKind document_kind, + deparse_expr_cxt* context +) { + appendStringInfoString(context->buf, "ifNull("); + deparseJsonbDocument(document, document_kind, context); + appendStringInfoString(context->buf, ", 'null')"); +} + static void deparseJsonbExists( Expr* document, @@ -5045,7 +5056,7 @@ deparseJsonbExists( if (document_kind == JSONB_DOCUMENT_STRING) { appendStringInfoString(buf, "if(NOT isValidJSON("); - deparseJsonbDocument(document, document_kind, context); + deparseJsonbNonnullDocument(document, document_kind, context); appendStringInfoString( buf, "), throwIf(1, 'invalid input syntax for type json'), " @@ -5053,13 +5064,13 @@ deparseJsonbExists( } appendStringInfoString(buf, "multiIf(JSONType("); - deparseJsonbDocument(document, document_kind, context); + deparseJsonbNonnullDocument(document, document_kind, context); appendStringInfoString(buf, ") = 'Object', JSONHas("); - deparseJsonbDocument(document, document_kind, context); + deparseJsonbNonnullDocument(document, document_kind, context); appendStringInfoString(buf, ", "); deparseExpr(key, context); appendStringInfoString(buf, "), JSONType("); - deparseJsonbDocument(document, document_kind, context); + deparseJsonbNonnullDocument(document, document_kind, context); appendStringInfoString( buf, ") = 'Array', arrayExists(jsonb_exists_element -> " @@ -5068,7 +5079,7 @@ deparseJsonbExists( ); deparseExpr(key, context); appendStringInfoString(buf, ", JSONExtractArrayRaw("); - deparseJsonbDocument(document, document_kind, context); + deparseJsonbNonnullDocument(document, document_kind, context); appendStringInfoString(buf, ")), 0)"); if (document_kind == JSONB_DOCUMENT_STRING) { diff --git a/test/expected/jsonb_exists.out b/test/expected/jsonb_exists.out index 741e31b6..c8ced30a 100644 --- a/test/expected/jsonb_exists.out +++ b/test/expected/jsonb_exists.out @@ -33,7 +33,7 @@ SELECT jsonb_exists_remote_sql( SELECT jsonb_exists_remote_sql( $$SELECT id FROM jsonb_exists_compatibility WHERE document ? 'key'$$ -) LIKE '%isValidJSON%'; +) LIKE '%isValidJSON(ifNull%'; ?column? ---------- t diff --git a/test/sql/jsonb_exists.sql b/test/sql/jsonb_exists.sql index a4280ba0..14c53e65 100644 --- a/test/sql/jsonb_exists.sql +++ b/test/sql/jsonb_exists.sql @@ -34,7 +34,7 @@ SELECT jsonb_exists_remote_sql( SELECT jsonb_exists_remote_sql( $$SELECT id FROM jsonb_exists_compatibility WHERE document ? 'key'$$ -) LIKE '%isValidJSON%'; +) LIKE '%isValidJSON(ifNull%'; SELECT jsonb_exists_remote_sql( $$SELECT id FROM jsonb_exists_compatibility From d13a5f6bc6e8c45a15e92ed3055938f1e9763844 Mon Sep 17 00:00:00 2001 From: Kostia R Date: Fri, 31 Jul 2026 16:06:20 +0000 Subject: [PATCH 3/4] Test JSONB existence against ClickHouse --- test/expected/jsonb_exists.out | 192 +++++++++++++++++++++++++-------- test/sql/jsonb_exists.sql | 91 ++++++++++------ 2 files changed, 209 insertions(+), 74 deletions(-) diff --git a/test/expected/jsonb_exists.out b/test/expected/jsonb_exists.out index c8ced30a..b711c601 100644 --- a/test/expected/jsonb_exists.out +++ b/test/expected/jsonb_exists.out @@ -1,65 +1,171 @@ CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS (dbname 'jsonb_exists', driver 'binary'); +CREATE SERVER CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +CREATE USER MAPPING +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS jsonb_exists'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query('CREATE DATABASE jsonb_exists'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + CREATE TABLE jsonb_exists.native_documents ( + id Int32, + document JSON + ) ENGINE = MergeTree ORDER BY id +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + INSERT INTO jsonb_exists.native_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '{"other":2}'), + (3, '{"other":null}') +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + CREATE TABLE jsonb_exists.string_documents ( + id Int32, + document Nullable(String) + ) ENGINE = MergeTree ORDER BY id +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + INSERT INTO jsonb_exists.string_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '["key","second",3]'), + (3, '{"other":2}'), + (4, '["other",3]'), + (5, NULL), + (6, '"key"') +$$); + clickhouse_raw_query +---------------------- + +(1 row) + CREATE FOREIGN TABLE jsonb_exists_native ( id integer, document jsonb ) SERVER jsonb_exists_binary OPTIONS (table_name 'native_documents'); +CREATE FOREIGN TABLE CREATE FOREIGN TABLE jsonb_exists_string ( id integer, document text ) SERVER jsonb_exists_binary OPTIONS (table_name 'string_documents'); +CREATE FOREIGN TABLE CREATE VIEW jsonb_exists_compatibility AS SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document FROM jsonb_exists_string; -CREATE FUNCTION jsonb_exists_remote_sql(query text) -RETURNS text -LANGUAGE plpgsql -AS $$ -DECLARE - plan jsonb; -BEGIN - EXECUTE 'EXPLAIN (FORMAT JSON, VERBOSE, COSTS OFF) ' || query INTO plan; - RETURN plan #>> '{0,Plan,Remote SQL}'; -END -$$; -SELECT jsonb_exists_remote_sql( - $$SELECT id FROM jsonb_exists_native WHERE document ? 'key'$$ -) LIKE '%JSONHas%'; - ?column? ----------- - t -(1 row) +CREATE VIEW +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_native + Output: id + Remote SQL: SELECT id FROM jsonb_exists.native_documents WHERE ((if(isNull(toJSONString(document)) OR isNull('key'), NULL, multiIf(JSONType(ifNull(toJSONString(document), 'null')) = 'Object', JSONHas(ifNull(toJSONString(document), 'null'), 'key'), JSONType(ifNull(toJSONString(document), 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'key', JSONExtractArrayRaw(ifNull(toJSONString(document), 'null'))), 0)))) ORDER BY id ASC NULLS LAST +(3 rows) -SELECT jsonb_exists_remote_sql( - $$SELECT id FROM jsonb_exists_compatibility WHERE document ? 'key'$$ -) LIKE '%isValidJSON(ifNull%'; - ?column? ----------- - t +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; + id +---- + 1 (1 row) -SELECT jsonb_exists_remote_sql( - $$SELECT id FROM jsonb_exists_compatibility - WHERE document ? 'first' OR document ? 'second'$$ -) LIKE '%JSONHas%JSONHas%'; - ?column? ----------- - t -(1 row) +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE ((if(isNull(document) OR isNull('key'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), throwIf(1, 'invalid input syntax for type json'), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'key'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'key', JSONExtractArrayRaw(ifNull(document, 'null'))), 0))))) ORDER BY id ASC NULLS LAST +(3 rows) -SELECT jsonb_exists_remote_sql( - $$SELECT id FROM jsonb_exists_compatibility - WHERE (document || '{}'::jsonb) ? 'key'$$ -) NOT LIKE '%JSONHas%'; - ?column? ----------- - t -(1 row) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + id +---- + 1 + 2 +(2 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE (((if(isNull(document) OR isNull('first'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), throwIf(1, 'invalid input syntax for type json'), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'first'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'first', JSONExtractArrayRaw(ifNull(document, 'null'))), 0)))) OR (if(isNull(document) OR isNull('second'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), throwIf(1, 'invalid input syntax for type json'), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'second'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'second', JSONExtractArrayRaw(ifNull(document, 'null'))), 0)))))) ORDER BY id ASC NULLS LAST +(3 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + id +---- + 1 + 2 +(2 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Filter: ((jsonb_in((jsonb_exists_string.document)::cstring) || '{}'::jsonb) ? 'key'::text) + Remote SQL: SELECT id, document FROM jsonb_exists.string_documents ORDER BY id ASC NULLS LAST +(4 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; + id +---- + 1 + 2 + 6 +(3 rows) DROP VIEW jsonb_exists_compatibility; -DROP FUNCTION jsonb_exists_remote_sql(text); +DROP VIEW DROP FOREIGN TABLE jsonb_exists_string; +DROP FOREIGN TABLE DROP FOREIGN TABLE jsonb_exists_native; -DROP SERVER jsonb_exists_binary CASCADE; -NOTICE: drop cascades to user mapping for ubuntu on server jsonb_exists_binary +DROP FOREIGN TABLE +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +DROP USER MAPPING +SELECT clickhouse_raw_query('DROP DATABASE jsonb_exists'); + clickhouse_raw_query +---------------------- + +(1 row) + +DROP SERVER jsonb_exists_binary; +DROP SERVER diff --git a/test/sql/jsonb_exists.sql b/test/sql/jsonb_exists.sql index 14c53e65..4ed6eeef 100644 --- a/test/sql/jsonb_exists.sql +++ b/test/sql/jsonb_exists.sql @@ -2,6 +2,36 @@ CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS (dbname 'jsonb_exists', driver 'binary'); CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS jsonb_exists'); +SELECT clickhouse_raw_query('CREATE DATABASE jsonb_exists'); +SELECT clickhouse_raw_query($$ + CREATE TABLE jsonb_exists.native_documents ( + id Int32, + document JSON + ) ENGINE = MergeTree ORDER BY id +$$); +SELECT clickhouse_raw_query($$ + INSERT INTO jsonb_exists.native_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '{"other":2}'), + (3, '{"other":null}') +$$); +SELECT clickhouse_raw_query($$ + CREATE TABLE jsonb_exists.string_documents ( + id Int32, + document Nullable(String) + ) ENGINE = MergeTree ORDER BY id +$$); +SELECT clickhouse_raw_query($$ + INSERT INTO jsonb_exists.string_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '["key","second",3]'), + (3, '{"other":2}'), + (4, '["other",3]'), + (5, NULL), + (6, '"key"') +$$); + CREATE FOREIGN TABLE jsonb_exists_native ( id integer, document jsonb @@ -16,38 +46,37 @@ CREATE VIEW jsonb_exists_compatibility AS SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document FROM jsonb_exists_string; -CREATE FUNCTION jsonb_exists_remote_sql(query text) -RETURNS text -LANGUAGE plpgsql -AS $$ -DECLARE - plan jsonb; -BEGIN - EXECUTE 'EXPLAIN (FORMAT JSON, VERBOSE, COSTS OFF) ' || query INTO plan; - RETURN plan #>> '{0,Plan,Remote SQL}'; -END -$$; - -SELECT jsonb_exists_remote_sql( - $$SELECT id FROM jsonb_exists_native WHERE document ? 'key'$$ -) LIKE '%JSONHas%'; - -SELECT jsonb_exists_remote_sql( - $$SELECT id FROM jsonb_exists_compatibility WHERE document ? 'key'$$ -) LIKE '%isValidJSON(ifNull%'; - -SELECT jsonb_exists_remote_sql( - $$SELECT id FROM jsonb_exists_compatibility - WHERE document ? 'first' OR document ? 'second'$$ -) LIKE '%JSONHas%JSONHas%'; - -SELECT jsonb_exists_remote_sql( - $$SELECT id FROM jsonb_exists_compatibility - WHERE (document || '{}'::jsonb) ? 'key'$$ -) NOT LIKE '%JSONHas%'; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; DROP VIEW jsonb_exists_compatibility; -DROP FUNCTION jsonb_exists_remote_sql(text); DROP FOREIGN TABLE jsonb_exists_string; DROP FOREIGN TABLE jsonb_exists_native; -DROP SERVER jsonb_exists_binary CASCADE; +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +SELECT clickhouse_raw_query('DROP DATABASE jsonb_exists'); +DROP SERVER jsonb_exists_binary; From 8f1948b60b4e89b2411dd806276709f4d82f346f Mon Sep 17 00:00:00 2001 From: Kostia R Date: Sat, 8 Aug 2026 10:40:47 +0000 Subject: [PATCH 4/4] Address JSONB existence review feedback --- CHANGELOG.md | 6 + doc/pg_clickhouse.md | 7 +- src/deparse.c | 150 ++++++++++++++++-------- test/expected/jsonb_exists.out | 158 +++++++++++++++---------- test/expected/jsonb_exists_1.out | 195 +++++++++++++++++++++++++++++++ test/expected/jsonb_exists_2.out | 195 +++++++++++++++++++++++++++++++ test/expected/result_map.txt | 13 +++ test/sql/jsonb_exists.sql | 76 +++++++++--- 8 files changed, 675 insertions(+), 125 deletions(-) create mode 100644 test/expected/jsonb_exists_1.out create mode 100644 test/expected/jsonb_exists_2.out diff --git a/CHANGELOG.md b/CHANGELOG.md index 024a42eb..b93afb88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,10 @@ All notable changes to this project will be documented in this file. It uses the * Coerce array elements in binary driver. `Array(Int32)` to `bigint[]`, or `quantilesExactLow()` results into `double precision[]`, no longer fails with `could not cast value from integer[] to bigint[]` ([#326]). +* Added pushdown for the JSONB `?` existence operator on native ClickHouse + JSON columns (24.8 and later) and compatibility views over JSON documents + stored in String columns (23.8 and later). Thanks to Kostia R for the PR + ([#325])! * Added support for the third argument to `array_position()` when it's a positive constant value, pushing it down to a call to `arraySlice()` to start the search from the specified index. Thanks to Minh Vu for the PR @@ -204,6 +208,8 @@ All notable changes to this project will be documented in this file. It uses the "ClickHouse/pg_clickhouse#317 Push down the array IN family unconditionally" [#319]: https://github.com/ClickHouse/pg_clickhouse/pull/319 "ClickHouse/pg_clickhouse#319 Fix foreign scan RTE selection" + [#325]: https://github.com/ClickHouse/pg_clickhouse/pull/325 + "ClickHouse/pg_clickhouse#325 Push down JSONB existence predicates" [#326]: https://github.com/ClickHouse/pg_clickhouse/pull/326 "ClickHouse/pg_clickhouse#326 pg-clickhouse-c" [#328]: https://github.com/ClickHouse/pg_clickhouse/pull/328 diff --git a/doc/pg_clickhouse.md b/doc/pg_clickhouse.md index e4902caf..39cda5e7 100644 --- a/doc/pg_clickhouse.md +++ b/doc/pg_clickhouse.md @@ -1347,7 +1347,12 @@ equivalents as follows: * `!~*` (case insensitive regexp not match): [match](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match) * `->>` (JSON/JSONB extract element as text): [sub-column syntax](https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns) * `->` (JSON/JSONB extract): [toJSONString](https://clickhouse.com/docs/sql-reference/functions/json-functions#toJSONString) + [sub-column syntax](https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns) -* `?` (JSONB top-level key or array string existence): [JSONHas](https://clickhouse.com/docs/sql-reference/functions/json-functions#jsonhas) for objects and [JSONExtractArrayRaw](https://clickhouse.com/docs/sql-reference/functions/json-functions#jsonextractarrayraw) for arrays. This also supports compatibility views that parse a JSON document stored in a ClickHouse `String`. +* `?` (JSONB top-level key or array string existence): + [JSONHas](https://clickhouse.com/docs/sql-reference/functions/json-functions#jsonhas) + for native JSON objects on ClickHouse 24.8 and later, and + [JSONExtractArrayRaw](https://clickhouse.com/docs/sql-reference/functions/json-functions#jsonextractarrayraw) + for arrays parsed from ClickHouse `String` columns by compatibility views + on ClickHouse 23.8 and later. ### IN and NULL Semantics diff --git a/src/deparse.c b/src/deparse.c index c3d49995..7e05cb96 100644 --- a/src/deparse.c +++ b/src/deparse.c @@ -212,6 +212,10 @@ static bool foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx); static bool is_shippable_subplan(SubPlan* subplan, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx); +static UserMapping* +foreign_expr_gate_user_mapping( + PlannerInfo* root, RelOptInfo* foreignrel, Oid serverid +); static char* deparse_type_name(Oid type_oid, int32 typemod); @@ -929,6 +933,12 @@ classifyJsonbDocument(Expr* expr, Expr** document) { return JSONB_DOCUMENT_NATIVE; } + /* + * Recognize the explicit jsonb_in() call used by compatibility views to + * expose a ClickHouse String column as PostgreSQL jsonb. Keep this special + * case scoped to jsonb_exists; general jsonb_in() pushdown can be added + * independently if other expressions need it. + */ if (IsA(expr, FuncExpr)) { FuncExpr* func = (FuncExpr*)expr; @@ -1157,6 +1167,42 @@ foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx) { JsonbDocumentKind document_kind = classifyJsonbDocument((Expr*)linitial(oe->args), &document); + if (document_kind != JSONB_DOCUMENT_UNSUPPORTED) { + UserMapping* user; + ch_server_version version; + + if (fpinfo == NULL || fpinfo->server == NULL) { + return false; + } + user = foreign_expr_gate_user_mapping( + glob_cxt->root, glob_cxt->foreignrel, fpinfo->server->serverid + ); + if (user == NULL) { + return false; + } + version = chfdw_get_server_version(user); + + /* + * Before 24.8, JSON was the experimental Object('json') type. + * It materializes every discovered path with a default value, + * so it cannot preserve top-level key existence. + */ + if (document_kind == JSONB_DOCUMENT_NATIVE && + !chfdw_version_ge(version, 24, 8)) { + return false; + } + + /* + * Before 23.8, ClickHouse's JSON validation did not accept all + * top-level scalar documents that PostgreSQL accepts. Evaluate + * compatibility views locally on those releases. + */ + if (document_kind == JSONB_DOCUMENT_STRING && + !chfdw_version_ge(version, 23, 8)) { + return false; + } + } + if (document_kind == JSONB_DOCUMENT_UNSUPPORTED || !foreign_expr_walker( (Node*)document, glob_cxt, EXPR_CTX_EXACT @@ -1540,8 +1586,8 @@ foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx) { * implemented. */ /* - * Resolve the user mapping the executor will use to scan foreignrel, for the - * plan-time server-version probe below. Mirrors the executor's own lookup + * Resolve the user mapping the executor will use to scan foreignrel for a + * plan-time server-version gate. Mirrors the executor's own lookup * (see clickhouseBeginForeignScan): the RTE's checkAsUser — set when the rel * is accessed on behalf of another user, e.g. through a view — wins over the * invoking user. Returns NULL instead of erroring when no mapping exists, so @@ -1549,7 +1595,9 @@ foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx) { * (or a bare EXPLAIN) at plan time. */ static UserMapping* -subplan_gate_user_mapping(PlannerInfo* root, RelOptInfo* foreignrel, Oid serverid) { +foreign_expr_gate_user_mapping( + PlannerInfo* root, RelOptInfo* foreignrel, Oid serverid +) { Oid userid = InvalidOid; int rtindex = -1; @@ -1813,7 +1861,7 @@ is_shippable_subplan(SubPlan* subplan, foreign_glob_cxt* glob_cxt, ExprTruthCtx * no mapping exists it refuses the pushdown rather than erroring. */ { - UserMapping* user = subplan_gate_user_mapping( + UserMapping* user = foreign_expr_gate_user_mapping( glob_cxt->root, glob_cxt->foreignrel, fpinfo->server->serverid ); @@ -5014,28 +5062,9 @@ findFunction(Oid typoid, char* name) { } static void -deparseJsonbDocument( - Expr* document, - JsonbDocumentKind document_kind, - deparse_expr_cxt* context -) { - if (document_kind == JSONB_DOCUMENT_NATIVE) { - appendStringInfoString(context->buf, "toJSONString("); - } - deparseExpr(document, context); - if (document_kind == JSONB_DOCUMENT_NATIVE) { - appendStringInfoChar(context->buf, ')'); - } -} - -static void -deparseJsonbNonnullDocument( - Expr* document, - JsonbDocumentKind document_kind, - deparse_expr_cxt* context -) { +deparseJsonbNonnullDocument(Expr* document, deparse_expr_cxt* context) { appendStringInfoString(context->buf, "ifNull("); - deparseJsonbDocument(document, document_kind, context); + deparseExpr(document, context); appendStringInfoString(context->buf, ", 'null')"); } @@ -5049,43 +5078,70 @@ deparseJsonbExists( StringInfo buf = context->buf; appendStringInfoString(buf, "(if(isNull("); - deparseJsonbDocument(document, document_kind, context); + deparseExpr(document, context); appendStringInfoString(buf, ") OR isNull("); deparseExpr(key, context); appendStringInfoString(buf, "), NULL, "); - if (document_kind == JSONB_DOCUMENT_STRING) { - appendStringInfoString(buf, "if(NOT isValidJSON("); - deparseJsonbNonnullDocument(document, document_kind, context); - appendStringInfoString( - buf, - "), throwIf(1, 'invalid input syntax for type json'), " - ); + if (document_kind == JSONB_DOCUMENT_NATIVE) { + appendStringInfoString(buf, "JSONHas("); + deparseExpr(document, context); + appendStringInfoString(buf, ", "); + deparseExpr(key, context); + appendStringInfoString(buf, ")))"); + return; } + appendStringInfoString(buf, "if(NOT isValidJSON("); + deparseJsonbNonnullDocument(document, context); + + /* + * throwIf() requires a constant message. Converting this prefixed value to + * UInt8 is guaranteed to fail while retaining the invalid document in the + * ClickHouse diagnostic. + */ + appendStringInfoString( + buf, + "), toUInt8(concat('invalid input syntax for type json: ', " + ); + deparseJsonbNonnullDocument(document, context); + appendStringInfoString(buf, ")), "); + appendStringInfoString(buf, "multiIf(JSONType("); - deparseJsonbNonnullDocument(document, document_kind, context); + deparseJsonbNonnullDocument(document, context); appendStringInfoString(buf, ") = 'Object', JSONHas("); - deparseJsonbNonnullDocument(document, document_kind, context); + deparseJsonbNonnullDocument(document, context); appendStringInfoString(buf, ", "); deparseExpr(key, context); appendStringInfoString(buf, "), JSONType("); - deparseJsonbNonnullDocument(document, document_kind, context); + deparseJsonbNonnullDocument(document, context); + + /* + * Keep the key expression outside the lambda body. A fixed lambda + * parameter could otherwise shadow a same-named column used as the key + * (for example, document ? jsonb_exists_element). Supply the key through + * a parallel, same-length array instead. + */ appendStringInfoString( buf, - ") = 'Array', arrayExists(jsonb_exists_element -> " - "JSONType(jsonb_exists_element) = 'String' AND " - "JSONExtractString(jsonb_exists_element) = " + ") = 'Array', arrayExists((__jsonb_element, __jsonb_key) -> " + "JSONType(__jsonb_element) = 'String' AND " + "JSONExtractString(__jsonb_element) = __jsonb_key, " + "JSONExtractArrayRaw(" ); + deparseJsonbNonnullDocument(document, context); + appendStringInfoString(buf, "), arrayWithConstant(length(JSONExtractArrayRaw("); + deparseJsonbNonnullDocument(document, context); + appendStringInfoString(buf, ")), "); deparseExpr(key, context); - appendStringInfoString(buf, ", JSONExtractArrayRaw("); - deparseJsonbNonnullDocument(document, document_kind, context); - appendStringInfoString(buf, ")), 0)"); - - if (document_kind == JSONB_DOCUMENT_STRING) { - appendStringInfoChar(buf, ')'); - } - appendStringInfoString(buf, "))"); + appendStringInfoString(buf, ")), JSONType("); + deparseJsonbNonnullDocument(document, context); + appendStringInfoString(buf, ") = 'String', JSONExtractString("); + deparseJsonbNonnullDocument(document, context); + appendStringInfoString(buf, ") = "); + deparseExpr(key, context); + appendStringInfoString(buf, ", 0)"); + appendStringInfoString(buf, ")))"); } /* diff --git a/test/expected/jsonb_exists.out b/test/expected/jsonb_exists.out index b711c601..1e17a25f 100644 --- a/test/expected/jsonb_exists.out +++ b/test/expected/jsonb_exists.out @@ -3,65 +3,47 @@ CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw CREATE SERVER CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; CREATE USER MAPPING -SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS jsonb_exists'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query('CREATE DATABASE jsonb_exists'); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query($$ +CREATE SERVER jsonb_exists_admin FOREIGN DATA WRAPPER clickhouse_fdw; +CREATE SERVER +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_admin; +CREATE USER MAPPING +CALL clickhouse_perform('jsonb_exists_admin', + 'DROP DATABASE IF EXISTS jsonb_exists'); +CALL +CALL clickhouse_perform('jsonb_exists_admin', 'CREATE DATABASE jsonb_exists'); +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ CREATE TABLE jsonb_exists.native_documents ( id Int32, document JSON ) ENGINE = MergeTree ORDER BY id $$); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query($$ +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ INSERT INTO jsonb_exists.native_documents VALUES (1, '{"key":1,"first":true}'), (2, '{"other":2}'), (3, '{"other":null}') $$); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query($$ +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ CREATE TABLE jsonb_exists.string_documents ( id Int32, - document Nullable(String) + document Nullable(String), + jsonb_exists_element String ) ENGINE = MergeTree ORDER BY id $$); - clickhouse_raw_query ----------------------- - -(1 row) - -SELECT clickhouse_raw_query($$ +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ INSERT INTO jsonb_exists.string_documents VALUES - (1, '{"key":1,"first":true}'), - (2, '["key","second",3]'), - (3, '{"other":2}'), - (4, '["other",3]'), - (5, NULL), - (6, '"key"') + (1, '{"key":1,"first":true}', 'key'), + (2, '["key","second",3]', 'key'), + (3, '{"other":2}', 'key'), + (4, '["other",3]', 'other'), + (5, NULL, 'key'), + (6, '"key"', 'key') $$); - clickhouse_raw_query ----------------------- - -(1 row) - +CALL CREATE FOREIGN TABLE jsonb_exists_native ( id integer, document jsonb @@ -69,20 +51,32 @@ CREATE FOREIGN TABLE jsonb_exists_native ( CREATE FOREIGN TABLE CREATE FOREIGN TABLE jsonb_exists_string ( id integer, - document text + document text, + jsonb_exists_element text ) SERVER jsonb_exists_binary OPTIONS (table_name 'string_documents'); CREATE FOREIGN TABLE CREATE VIEW jsonb_exists_compatibility AS -SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document +SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document, + jsonb_exists_element FROM jsonb_exists_string; CREATE VIEW +WITH version AS ( + SELECT string_to_array( + clickhouse_server_version('jsonb_exists_binary'), '.' + )::int[] AS parts +) +SELECT + (parts[1], parts[2]) >= (24, 8) AS ch_has_json_type, + (parts[1], parts[2]) >= (23, 8) AS ch_has_json_validation +FROM version \gset +\if :ch_has_json_type EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- Foreign Scan on public.jsonb_exists_native Output: id - Remote SQL: SELECT id FROM jsonb_exists.native_documents WHERE ((if(isNull(toJSONString(document)) OR isNull('key'), NULL, multiIf(JSONType(ifNull(toJSONString(document), 'null')) = 'Object', JSONHas(ifNull(toJSONString(document), 'null'), 'key'), JSONType(ifNull(toJSONString(document), 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'key', JSONExtractArrayRaw(ifNull(toJSONString(document), 'null'))), 0)))) ORDER BY id ASC NULLS LAST + Remote SQL: SELECT id FROM jsonb_exists.native_documents WHERE ((if(isNull(document) OR isNull('key'), NULL, JSONHas(document, 'key')))) ORDER BY id ASC NULLS LAST (3 rows) SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; @@ -91,15 +85,16 @@ SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; 1 (1 row) +\endif EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM jsonb_exists_compatibility WHERE document ? 'key' ORDER BY id; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Foreign Scan on public.jsonb_exists_string Output: jsonb_exists_string.id - Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE ((if(isNull(document) OR isNull('key'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), throwIf(1, 'invalid input syntax for type json'), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'key'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'key', JSONExtractArrayRaw(ifNull(document, 'null'))), 0))))) ORDER BY id ASC NULLS LAST + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE ((if(isNull(document) OR isNull('key'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), toUInt8(concat('invalid input syntax for type json: ', ifNull(document, 'null'))), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'key'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists((__jsonb_element, __jsonb_key) -> JSONType(__jsonb_element) = 'String' AND JSONExtractString(__jsonb_element) = __jsonb_key, JSONExtractArrayRaw(ifNull(document, 'null')), arrayWithConstant(length(JSONExtractArrayRaw(ifNull(document, 'null'))), 'key')), JSONType(ifNull(document, 'null')) = 'String', JSONExtractString(ifNull(document, 'null')) = 'key', 0))))) ORDER BY id ASC NULLS LAST (3 rows) SELECT id FROM jsonb_exists_compatibility @@ -109,17 +104,18 @@ ORDER BY id; ---- 1 2 -(2 rows) + 6 +(3 rows) EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM jsonb_exists_compatibility WHERE document ? 'first' OR document ? 'second' ORDER BY id; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Foreign Scan on public.jsonb_exists_string Output: jsonb_exists_string.id - Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE (((if(isNull(document) OR isNull('first'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), throwIf(1, 'invalid input syntax for type json'), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'first'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'first', JSONExtractArrayRaw(ifNull(document, 'null'))), 0)))) OR (if(isNull(document) OR isNull('second'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), throwIf(1, 'invalid input syntax for type json'), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'second'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'second', JSONExtractArrayRaw(ifNull(document, 'null'))), 0)))))) ORDER BY id ASC NULLS LAST + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE (((if(isNull(document) OR isNull('first'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), toUInt8(concat('invalid input syntax for type json: ', ifNull(document, 'null'))), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'first'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists((__jsonb_element, __jsonb_key) -> JSONType(__jsonb_element) = 'String' AND JSONExtractString(__jsonb_element) = __jsonb_key, JSONExtractArrayRaw(ifNull(document, 'null')), arrayWithConstant(length(JSONExtractArrayRaw(ifNull(document, 'null'))), 'first')), JSONType(ifNull(document, 'null')) = 'String', JSONExtractString(ifNull(document, 'null')) = 'first', 0)))) OR (if(isNull(document) OR isNull('second'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), toUInt8(concat('invalid input syntax for type json: ', ifNull(document, 'null'))), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'second'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists((__jsonb_element, __jsonb_key) -> JSONType(__jsonb_element) = 'String' AND JSONExtractString(__jsonb_element) = __jsonb_key, JSONExtractArrayRaw(ifNull(document, 'null')), arrayWithConstant(length(JSONExtractArrayRaw(ifNull(document, 'null'))), 'second')), JSONType(ifNull(document, 'null')) = 'String', JSONExtractString(ifNull(document, 'null')) = 'second', 0)))))) ORDER BY id ASC NULLS LAST (3 rows) SELECT id FROM jsonb_exists_compatibility @@ -131,6 +127,28 @@ ORDER BY id; 2 (2 rows) +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? jsonb_exists_element +ORDER BY id; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE ((if(isNull(document) OR isNull(jsonb_exists_element), NULL, if(NOT isValidJSON(ifNull(document, 'null')), toUInt8(concat('invalid input syntax for type json: ', ifNull(document, 'null'))), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), jsonb_exists_element), JSONType(ifNull(document, 'null')) = 'Array', arrayExists((__jsonb_element, __jsonb_key) -> JSONType(__jsonb_element) = 'String' AND JSONExtractString(__jsonb_element) = __jsonb_key, JSONExtractArrayRaw(ifNull(document, 'null')), arrayWithConstant(length(JSONExtractArrayRaw(ifNull(document, 'null'))), jsonb_exists_element)), JSONType(ifNull(document, 'null')) = 'String', JSONExtractString(ifNull(document, 'null')) = jsonb_exists_element, 0))))) ORDER BY id ASC NULLS LAST +(3 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? jsonb_exists_element +ORDER BY id; + id +---- + 1 + 2 + 4 + 6 +(4 rows) + EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM jsonb_exists_compatibility WHERE (document || '{}'::jsonb) ? 'key' @@ -153,6 +171,24 @@ ORDER BY id; 6 (3 rows) +\if :ch_has_json_validation +CALL clickhouse_perform('jsonb_exists_admin', $$ + INSERT INTO jsonb_exists.string_documents VALUES (7, 'not json', 'key') +$$); +CALL +DO $$ +BEGIN + PERFORM id FROM jsonb_exists_compatibility + WHERE id = 7 AND document ? 'key'; + RAISE NOTICE 'invalid JSON error includes document: f'; +EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'invalid JSON error includes document: %', + position('not json' IN SQLERRM) > 0; +END +$$; +NOTICE: invalid JSON error includes document: t +DO +\endif DROP VIEW jsonb_exists_compatibility; DROP VIEW DROP FOREIGN TABLE jsonb_exists_string; @@ -161,11 +197,11 @@ DROP FOREIGN TABLE jsonb_exists_native; DROP FOREIGN TABLE DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; DROP USER MAPPING -SELECT clickhouse_raw_query('DROP DATABASE jsonb_exists'); - clickhouse_raw_query ----------------------- - -(1 row) - +CALL clickhouse_perform('jsonb_exists_admin', 'DROP DATABASE jsonb_exists'); +CALL +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_admin; +DROP USER MAPPING DROP SERVER jsonb_exists_binary; DROP SERVER +DROP SERVER jsonb_exists_admin; +DROP SERVER diff --git a/test/expected/jsonb_exists_1.out b/test/expected/jsonb_exists_1.out new file mode 100644 index 00000000..5e364ebe --- /dev/null +++ b/test/expected/jsonb_exists_1.out @@ -0,0 +1,195 @@ +CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw + OPTIONS (dbname 'jsonb_exists', driver 'binary'); +CREATE SERVER +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +CREATE USER MAPPING +CREATE SERVER jsonb_exists_admin FOREIGN DATA WRAPPER clickhouse_fdw; +CREATE SERVER +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_admin; +CREATE USER MAPPING +CALL clickhouse_perform('jsonb_exists_admin', + 'DROP DATABASE IF EXISTS jsonb_exists'); +CALL +CALL clickhouse_perform('jsonb_exists_admin', 'CREATE DATABASE jsonb_exists'); +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ + CREATE TABLE jsonb_exists.native_documents ( + id Int32, + document JSON + ) ENGINE = MergeTree ORDER BY id +$$); +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ + INSERT INTO jsonb_exists.native_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '{"other":2}'), + (3, '{"other":null}') +$$); +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ + CREATE TABLE jsonb_exists.string_documents ( + id Int32, + document Nullable(String), + jsonb_exists_element String + ) ENGINE = MergeTree ORDER BY id +$$); +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ + INSERT INTO jsonb_exists.string_documents VALUES + (1, '{"key":1,"first":true}', 'key'), + (2, '["key","second",3]', 'key'), + (3, '{"other":2}', 'key'), + (4, '["other",3]', 'other'), + (5, NULL, 'key'), + (6, '"key"', 'key') +$$); +CALL +CREATE FOREIGN TABLE jsonb_exists_native ( + id integer, + document jsonb +) SERVER jsonb_exists_binary OPTIONS (table_name 'native_documents'); +CREATE FOREIGN TABLE +CREATE FOREIGN TABLE jsonb_exists_string ( + id integer, + document text, + jsonb_exists_element text +) SERVER jsonb_exists_binary OPTIONS (table_name 'string_documents'); +CREATE FOREIGN TABLE +CREATE VIEW jsonb_exists_compatibility AS +SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document, + jsonb_exists_element +FROM jsonb_exists_string; +CREATE VIEW +WITH version AS ( + SELECT string_to_array( + clickhouse_server_version('jsonb_exists_binary'), '.' + )::int[] AS parts +) +SELECT + (parts[1], parts[2]) >= (24, 8) AS ch_has_json_type, + (parts[1], parts[2]) >= (23, 8) AS ch_has_json_validation +FROM version \gset +\if :ch_has_json_type +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; +\endif +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE ((if(isNull(document) OR isNull('key'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), toUInt8(concat('invalid input syntax for type json: ', ifNull(document, 'null'))), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'key'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists((__jsonb_element, __jsonb_key) -> JSONType(__jsonb_element) = 'String' AND JSONExtractString(__jsonb_element) = __jsonb_key, JSONExtractArrayRaw(ifNull(document, 'null')), arrayWithConstant(length(JSONExtractArrayRaw(ifNull(document, 'null'))), 'key')), JSONType(ifNull(document, 'null')) = 'String', JSONExtractString(ifNull(document, 'null')) = 'key', 0))))) ORDER BY id ASC NULLS LAST +(3 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + id +---- + 1 + 2 + 6 +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE (((if(isNull(document) OR isNull('first'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), toUInt8(concat('invalid input syntax for type json: ', ifNull(document, 'null'))), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'first'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists((__jsonb_element, __jsonb_key) -> JSONType(__jsonb_element) = 'String' AND JSONExtractString(__jsonb_element) = __jsonb_key, JSONExtractArrayRaw(ifNull(document, 'null')), arrayWithConstant(length(JSONExtractArrayRaw(ifNull(document, 'null'))), 'first')), JSONType(ifNull(document, 'null')) = 'String', JSONExtractString(ifNull(document, 'null')) = 'first', 0)))) OR (if(isNull(document) OR isNull('second'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), toUInt8(concat('invalid input syntax for type json: ', ifNull(document, 'null'))), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'second'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists((__jsonb_element, __jsonb_key) -> JSONType(__jsonb_element) = 'String' AND JSONExtractString(__jsonb_element) = __jsonb_key, JSONExtractArrayRaw(ifNull(document, 'null')), arrayWithConstant(length(JSONExtractArrayRaw(ifNull(document, 'null'))), 'second')), JSONType(ifNull(document, 'null')) = 'String', JSONExtractString(ifNull(document, 'null')) = 'second', 0)))))) ORDER BY id ASC NULLS LAST +(3 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + id +---- + 1 + 2 +(2 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? jsonb_exists_element +ORDER BY id; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE ((if(isNull(document) OR isNull(jsonb_exists_element), NULL, if(NOT isValidJSON(ifNull(document, 'null')), toUInt8(concat('invalid input syntax for type json: ', ifNull(document, 'null'))), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), jsonb_exists_element), JSONType(ifNull(document, 'null')) = 'Array', arrayExists((__jsonb_element, __jsonb_key) -> JSONType(__jsonb_element) = 'String' AND JSONExtractString(__jsonb_element) = __jsonb_key, JSONExtractArrayRaw(ifNull(document, 'null')), arrayWithConstant(length(JSONExtractArrayRaw(ifNull(document, 'null'))), jsonb_exists_element)), JSONType(ifNull(document, 'null')) = 'String', JSONExtractString(ifNull(document, 'null')) = jsonb_exists_element, 0))))) ORDER BY id ASC NULLS LAST +(3 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? jsonb_exists_element +ORDER BY id; + id +---- + 1 + 2 + 4 + 6 +(4 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Filter: ((jsonb_in((jsonb_exists_string.document)::cstring) || '{}'::jsonb) ? 'key'::text) + Remote SQL: SELECT id, document FROM jsonb_exists.string_documents ORDER BY id ASC NULLS LAST +(4 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; + id +---- + 1 + 2 + 6 +(3 rows) + +\if :ch_has_json_validation +CALL clickhouse_perform('jsonb_exists_admin', $$ + INSERT INTO jsonb_exists.string_documents VALUES (7, 'not json', 'key') +$$); +CALL +DO $$ +BEGIN + PERFORM id FROM jsonb_exists_compatibility + WHERE id = 7 AND document ? 'key'; + RAISE NOTICE 'invalid JSON error includes document: f'; +EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'invalid JSON error includes document: %', + position('not json' IN SQLERRM) > 0; +END +$$; +NOTICE: invalid JSON error includes document: t +DO +\endif +DROP VIEW jsonb_exists_compatibility; +DROP VIEW +DROP FOREIGN TABLE jsonb_exists_string; +DROP FOREIGN TABLE +DROP FOREIGN TABLE jsonb_exists_native; +DROP FOREIGN TABLE +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +DROP USER MAPPING +CALL clickhouse_perform('jsonb_exists_admin', 'DROP DATABASE jsonb_exists'); +CALL +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_admin; +DROP USER MAPPING +DROP SERVER jsonb_exists_binary; +DROP SERVER +DROP SERVER jsonb_exists_admin; +DROP SERVER diff --git a/test/expected/jsonb_exists_2.out b/test/expected/jsonb_exists_2.out new file mode 100644 index 00000000..5dcc99d3 --- /dev/null +++ b/test/expected/jsonb_exists_2.out @@ -0,0 +1,195 @@ +CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw + OPTIONS (dbname 'jsonb_exists', driver 'binary'); +CREATE SERVER +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +CREATE USER MAPPING +CREATE SERVER jsonb_exists_admin FOREIGN DATA WRAPPER clickhouse_fdw; +CREATE SERVER +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_admin; +CREATE USER MAPPING +CALL clickhouse_perform('jsonb_exists_admin', + 'DROP DATABASE IF EXISTS jsonb_exists'); +CALL +CALL clickhouse_perform('jsonb_exists_admin', 'CREATE DATABASE jsonb_exists'); +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ + CREATE TABLE jsonb_exists.native_documents ( + id Int32, + document JSON + ) ENGINE = MergeTree ORDER BY id +$$); +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ + INSERT INTO jsonb_exists.native_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '{"other":2}'), + (3, '{"other":null}') +$$); +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ + CREATE TABLE jsonb_exists.string_documents ( + id Int32, + document Nullable(String), + jsonb_exists_element String + ) ENGINE = MergeTree ORDER BY id +$$); +CALL +CALL clickhouse_perform('jsonb_exists_admin', $$ + INSERT INTO jsonb_exists.string_documents VALUES + (1, '{"key":1,"first":true}', 'key'), + (2, '["key","second",3]', 'key'), + (3, '{"other":2}', 'key'), + (4, '["other",3]', 'other'), + (5, NULL, 'key'), + (6, '"key"', 'key') +$$); +CALL +CREATE FOREIGN TABLE jsonb_exists_native ( + id integer, + document jsonb +) SERVER jsonb_exists_binary OPTIONS (table_name 'native_documents'); +CREATE FOREIGN TABLE +CREATE FOREIGN TABLE jsonb_exists_string ( + id integer, + document text, + jsonb_exists_element text +) SERVER jsonb_exists_binary OPTIONS (table_name 'string_documents'); +CREATE FOREIGN TABLE +CREATE VIEW jsonb_exists_compatibility AS +SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document, + jsonb_exists_element +FROM jsonb_exists_string; +CREATE VIEW +WITH version AS ( + SELECT string_to_array( + clickhouse_server_version('jsonb_exists_binary'), '.' + )::int[] AS parts +) +SELECT + (parts[1], parts[2]) >= (24, 8) AS ch_has_json_type, + (parts[1], parts[2]) >= (23, 8) AS ch_has_json_validation +FROM version \gset +\if :ch_has_json_type +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; +\endif +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Filter: (jsonb_in((jsonb_exists_string.document)::cstring) ? 'key'::text) + Remote SQL: SELECT id, document FROM jsonb_exists.string_documents ORDER BY id ASC NULLS LAST +(4 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + id +---- + 1 + 2 + 6 +(3 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Filter: ((jsonb_in((jsonb_exists_string.document)::cstring) ? 'first'::text) OR (jsonb_in((jsonb_exists_string.document)::cstring) ? 'second'::text)) + Remote SQL: SELECT id, document FROM jsonb_exists.string_documents ORDER BY id ASC NULLS LAST +(4 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + id +---- + 1 + 2 +(2 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? jsonb_exists_element +ORDER BY id; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Filter: (jsonb_in((jsonb_exists_string.document)::cstring) ? jsonb_exists_string.jsonb_exists_element) + Remote SQL: SELECT id, document, jsonb_exists_element FROM jsonb_exists.string_documents ORDER BY id ASC NULLS LAST +(4 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? jsonb_exists_element +ORDER BY id; + id +---- + 1 + 2 + 4 + 6 +(4 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Filter: ((jsonb_in((jsonb_exists_string.document)::cstring) || '{}'::jsonb) ? 'key'::text) + Remote SQL: SELECT id, document FROM jsonb_exists.string_documents ORDER BY id ASC NULLS LAST +(4 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; + id +---- + 1 + 2 + 6 +(3 rows) + +\if :ch_has_json_validation +CALL clickhouse_perform('jsonb_exists_admin', $$ + INSERT INTO jsonb_exists.string_documents VALUES (7, 'not json', 'key') +$$); +DO $$ +BEGIN + PERFORM id FROM jsonb_exists_compatibility + WHERE id = 7 AND document ? 'key'; + RAISE NOTICE 'invalid JSON error includes document: f'; +EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'invalid JSON error includes document: %', + position('not json' IN SQLERRM) > 0; +END +$$; +\endif +DROP VIEW jsonb_exists_compatibility; +DROP VIEW +DROP FOREIGN TABLE jsonb_exists_string; +DROP FOREIGN TABLE +DROP FOREIGN TABLE jsonb_exists_native; +DROP FOREIGN TABLE +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +DROP USER MAPPING +CALL clickhouse_perform('jsonb_exists_admin', 'DROP DATABASE jsonb_exists'); +CALL +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_admin; +DROP USER MAPPING +DROP SERVER jsonb_exists_binary; +DROP SERVER +DROP SERVER jsonb_exists_admin; +DROP SERVER diff --git a/test/expected/result_map.txt b/test/expected/result_map.txt index 68cf4b67..027cefce 100644 --- a/test/expected/result_map.txt +++ b/test/expected/result_map.txt @@ -221,6 +221,19 @@ json.sql 23.8 | json_5.out 23.3 | json_6.out +jsonb_exists.sql +---------------- + + Postgres | File +----------|------------------ + 13+ | jsonb_exists.out + + ClickHouse | File +------------|-------------------- + 24.8+ | jsonb_exists.out + 23.8-24.3 | jsonb_exists_1.out + 23.3 | jsonb_exists_2.out + param.sql --------- diff --git a/test/sql/jsonb_exists.sql b/test/sql/jsonb_exists.sql index 4ed6eeef..1905dc6c 100644 --- a/test/sql/jsonb_exists.sql +++ b/test/sql/jsonb_exists.sql @@ -2,34 +2,39 @@ CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS (dbname 'jsonb_exists', driver 'binary'); CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; -SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS jsonb_exists'); -SELECT clickhouse_raw_query('CREATE DATABASE jsonb_exists'); -SELECT clickhouse_raw_query($$ +CREATE SERVER jsonb_exists_admin FOREIGN DATA WRAPPER clickhouse_fdw; +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_admin; + +CALL clickhouse_perform('jsonb_exists_admin', + 'DROP DATABASE IF EXISTS jsonb_exists'); +CALL clickhouse_perform('jsonb_exists_admin', 'CREATE DATABASE jsonb_exists'); +CALL clickhouse_perform('jsonb_exists_admin', $$ CREATE TABLE jsonb_exists.native_documents ( id Int32, document JSON ) ENGINE = MergeTree ORDER BY id $$); -SELECT clickhouse_raw_query($$ +CALL clickhouse_perform('jsonb_exists_admin', $$ INSERT INTO jsonb_exists.native_documents VALUES (1, '{"key":1,"first":true}'), (2, '{"other":2}'), (3, '{"other":null}') $$); -SELECT clickhouse_raw_query($$ +CALL clickhouse_perform('jsonb_exists_admin', $$ CREATE TABLE jsonb_exists.string_documents ( id Int32, - document Nullable(String) + document Nullable(String), + jsonb_exists_element String ) ENGINE = MergeTree ORDER BY id $$); -SELECT clickhouse_raw_query($$ +CALL clickhouse_perform('jsonb_exists_admin', $$ INSERT INTO jsonb_exists.string_documents VALUES - (1, '{"key":1,"first":true}'), - (2, '["key","second",3]'), - (3, '{"other":2}'), - (4, '["other",3]'), - (5, NULL), - (6, '"key"') + (1, '{"key":1,"first":true}', 'key'), + (2, '["key","second",3]', 'key'), + (3, '{"other":2}', 'key'), + (4, '["other",3]', 'other'), + (5, NULL, 'key'), + (6, '"key"', 'key') $$); CREATE FOREIGN TABLE jsonb_exists_native ( @@ -39,16 +44,29 @@ CREATE FOREIGN TABLE jsonb_exists_native ( CREATE FOREIGN TABLE jsonb_exists_string ( id integer, - document text + document text, + jsonb_exists_element text ) SERVER jsonb_exists_binary OPTIONS (table_name 'string_documents'); CREATE VIEW jsonb_exists_compatibility AS -SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document +SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document, + jsonb_exists_element FROM jsonb_exists_string; +WITH version AS ( + SELECT string_to_array( + clickhouse_server_version('jsonb_exists_binary'), '.' + )::int[] AS parts +) +SELECT + (parts[1], parts[2]) >= (24, 8) AS ch_has_json_type, + (parts[1], parts[2]) >= (23, 8) AS ch_has_json_validation +FROM version \gset +\if :ch_has_json_type EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; +\endif EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM jsonb_exists_compatibility @@ -66,6 +84,14 @@ SELECT id FROM jsonb_exists_compatibility WHERE document ? 'first' OR document ? 'second' ORDER BY id; +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? jsonb_exists_element +ORDER BY id; +SELECT id FROM jsonb_exists_compatibility +WHERE document ? jsonb_exists_element +ORDER BY id; + EXPLAIN (VERBOSE, COSTS OFF) SELECT id FROM jsonb_exists_compatibility WHERE (document || '{}'::jsonb) ? 'key' @@ -74,9 +100,27 @@ SELECT id FROM jsonb_exists_compatibility WHERE (document || '{}'::jsonb) ? 'key' ORDER BY id; +\if :ch_has_json_validation +CALL clickhouse_perform('jsonb_exists_admin', $$ + INSERT INTO jsonb_exists.string_documents VALUES (7, 'not json', 'key') +$$); +DO $$ +BEGIN + PERFORM id FROM jsonb_exists_compatibility + WHERE id = 7 AND document ? 'key'; + RAISE NOTICE 'invalid JSON error includes document: f'; +EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'invalid JSON error includes document: %', + position('not json' IN SQLERRM) > 0; +END +$$; +\endif + DROP VIEW jsonb_exists_compatibility; DROP FOREIGN TABLE jsonb_exists_string; DROP FOREIGN TABLE jsonb_exists_native; DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; -SELECT clickhouse_raw_query('DROP DATABASE jsonb_exists'); +CALL clickhouse_perform('jsonb_exists_admin', 'DROP DATABASE jsonb_exists'); +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_admin; DROP SERVER jsonb_exists_binary; +DROP SERVER jsonb_exists_admin;