fix(sql): allow bounded JSON text in existing functional indexes

docs/design/20260929-functional-index-initial-support.md

@@ -30,6 +30,32 @@ their direct references; only expression parts acquire hidden generated values.
 Unique functional keys, directions, prefixes, INCLUDE columns and
 special index algorithms are unsupported.
 
+### Bounded JSON text eligibility extension
+
+The follow-up in #28274 only extends this DDL allowlist. An ordinary JSON
+column may feed `JSON_EXTRACT` with a constant, single-location path, followed
+by `JSON_UNQUOTE` and an explicit positive-width `CAST(... AS CHAR(n))`.
+The resulting key is ordinary bounded text; neither JSON columns nor
+JSON_EXTRACT results are admitted as physical JSON index keys. Dynamic paths,
+wildcards, oversized array offsets, generated JSON sources and arbitrary JSON
+producers remain outside this extension. The path must pass both the restricted
+grammar and the existing ByteJSON parser before metadata publication.
+
+The existing function registry, JSON execution, CAST semantics, hidden-column
+metadata and secondary-index maintenance remain unchanged. No MIN/MAX,
+ODKU result tracking, wire fields, JSON renderer, vector/batch state or new
+runtime conversion policy belongs to this extension. It uses existing function
+identities and the existing protocol-107 functional-index representation;
+the removed cumulative branch's JSON/ODKU protocol chain is not a dependency.
+Old coordinators retain their conservative DDL/metadata rejection of this
+expression. This does not claim mixed-binary upgrade or deployed QA acceptance.
+
+Equality matching may unwrap only the binder's value-preserving CHAR-to-VARCHAR
+comparison cast. The original predicate remains as a base-row residual. The
+validation matrix includes rejected raw JSON keys, existing arithmetic/LOWER
+controls, same-expression physical-index selection, write/update/rollback,
+LIKE/COPY replay and hidden-column cleanup.
+
 ## Ownership and execution
 
 Each expression part owns a hidden virtual generated column named from the

pkg/sql/plan/functional_index.go

@@ -130,6 +130,9 @@ func functionalExpressionEligible(expr *Expr, cols []*ColDef, visiting map[int32
 		if e.F == nil || e.F.Func == nil {
 			return false
 		}
+		if functionalJSONTextCastEligible(expr, cols) {
+			return true
+		}
 		switch e.F.Func.ObjName {
 		case "lower", "upper":
 			if len(e.F.Args) != 1 || !functionalTextType(e.F.Args[0].Typ) || !functionalTextType(expr.Typ) {
@@ -311,6 +314,7 @@ func validateFunctionalTable(ctx context.Context, table *TableDef) error {
 			}
 			col := functionalIndexPartColumn(table, index, ordinal)
 			if col == nil || col.GeneratedCol.IsStored || col.GeneratedCol.Expr == nil || col.GeneratedCol.OriginString == "" ||
+				(!types.T(col.Typ.Id).IsInteger() && !functionalTextType(col.Typ)) ||
 				!sameFunctionalValueType(col.Typ, col.GeneratedCol.Expr.Typ) ||
 				!functionalExpressionEligible(col.GeneratedCol.Expr, table.Cols, make(map[int32]bool)) {
 				return moerr.NewInvalidInput(ctx, "invalid functional index metadata")
@@ -431,7 +435,7 @@ func (builder *QueryBuilder) addFunctionalIndexFilters(node *pb.Node) {
 					if constant.GetLit() == nil || constant.GetLit().Isnull {
 						continue
 					}
-					candidate := DeepCopyExpr(f.Args[side])
+					candidate := DeepCopyExpr(functionalJSONComparisonOperand(f.Args[side], node.TableDef.Cols, node.BindingTags[0]))
 					persisted := DeepCopyExpr(col.GeneratedCol.Expr)
 					if !normalizeFunctionalExpression(candidate, node.BindingTags[0]) || !normalizeFunctionalExpression(persisted, 0) ||
 						!functionalExpressionEligible(candidate, node.TableDef.Cols, make(map[int32]bool)) || !proto.Equal(candidate, persisted) {

pkg/sql/plan/functional_json_index.go

@@ -0,0 +1,126 @@
+// Copyright 2026 Matrix Origin
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package plan
+
+import (
+	"context"
+	"regexp"
+	"strings"
+
+	"github.com/matrixorigin/matrixone/pkg/container/bytejson"
+	"github.com/matrixorigin/matrixone/pkg/container/types"
+	"github.com/matrixorigin/matrixone/pkg/sql/plan/function"
+)
+
+var functionalJSONPath = regexp.MustCompile(`^\$(?:\.[A-Za-z_][A-Za-z0-9_]*|\[[0-9]+\])*$`)
+
+// This only extends DDL eligibility. The existing JSON functions, text CAST,
+// hidden generated column and BTREE owners retain all execution semantics.
+// In particular, no JSON value is admitted as a physical index key.
+func functionalJSONTextCastEligible(expr *Expr, cols []*ColDef) bool {
+	if expr == nil || !functionalTextType(expr.Typ) || expr.Typ.Width <= 0 ||
+		expr.Typ.ValidateCollation() != nil || expr.Typ.CollationVersion != 0 ||
+		expr.Typ.Charset > uint32(types.CharsetUTF8) || !functionalJSONFunction(expr, "cast", function.CAST, 1) {
+		return false
+	}
+	args := expr.GetF().Args
+	if len(args) != 2 || args[1].GetT() == nil || !sameFunctionalValueType(expr.Typ, args[1].Typ) {
+		return false
+	}
+	unquote := args[0]
+	if !functionalJSONFunction(unquote, "json_unquote", function.JSON_UNQUOTE, 0) ||
+		len(unquote.GetF().Args) != 1 {
+		return false
+	}
+	extract := unquote.GetF().Args[0]
+	if !functionalJSONFunction(extract, "json_extract", function.JSON_EXTRACT, 0) ||
+		extract.Typ.Id != int32(types.T_json) || len(extract.GetF().Args) != 2 {
+		return false
+	}
+	source, path := extract.GetF().Args[0], extract.GetF().Args[1]
+	if source == nil || path == nil || path.GetLit() == nil || path.GetLit().Isnull ||
+		!functionalJSONPath.MatchString(path.GetLit().GetSval()) {
+		return false
+	}
+	if _, err := bytejson.ParseJsonPath(path.GetLit().GetSval()); err != nil {
+		return false
+	}
+	// Keep this extension independent of generated-column/FK conversion and
+	// session-sensitive JSON producers. Only an ordinary JSON column is added.
+	ref := source.GetCol()
+	if ref == nil || ref.RelPos != 0 || ref.ColPos < 0 || int(ref.ColPos) >= len(cols) {
+		return false
+	}
+	col := cols[ref.ColPos]
+	return col != nil && !col.Hidden && !col.Typ.AutoIncr && col.GeneratedCol == nil &&
+		col.Typ.Id == int32(types.T_json) && sameFunctionalValueType(source.Typ, col.Typ)
+}
+
+func functionalJSONFunction(expr *Expr, name string, id, overload int32) bool {
+	if expr == nil || expr.Typ.ValidateCollation() != nil {
+		return false
+	}
+	fn := expr.GetF()
+	if fn == nil || fn.Func == nil || fn.Func.ObjName != name ||
+		fn.Func.Obj != int64(id)<<32|int64(overload) {
+		return false
+	}
+	args := make([]types.Type, len(fn.Args))
+	for i, arg := range fn.Args {
+		if arg == nil || arg.Typ.ValidateCollation() != nil || strings.HasPrefix(types.T(arg.Typ.Id).String(), "unexpected type:") {
+			return false
+		}
+		var err error
+		args[i], err = types.TypeFromPlan(arg.Typ)
+		if err != nil {
+			return false
+		}
+	}
+	resolved, err := function.GetFunctionByNameWithOverload(context.Background(), name, args, overload)
+	if err != nil || resolved.GetEncodedOverloadID() != fn.Func.Obj {
+		return false
+	}
+	if targets, needed := resolved.ShouldDoImplicitTypeCast(); needed {
+		for i := range args {
+			if len(targets) != len(args) || !args[i].Eq(targets[i]) {
+				return false
+			}
+		}
+	}
+	return id == function.CAST || sameFunctionalValueType(resolved.GetReturnType().PlanType(), expr.Typ)
+}
+
+// Equality widens a CHAR operand to VARCHAR. Strip only that value-preserving
+// wrapper for matching; keep the original base-row predicate as a residual.
+func functionalJSONComparisonOperand(expr *Expr, cols []*ColDef, tag int32) *Expr {
+	if !functionalJSONFunction(expr, "cast", function.CAST, 2) || len(expr.GetF().Args) != 2 {
+		return expr
+	}
+	inner := expr.GetF().Args[0]
+	from, to := inner.Typ, expr.Typ
+	if expr.GetF().Args[1].GetT() == nil || !sameFunctionalValueType(to, expr.GetF().Args[1].Typ) ||
+		from.Id != int32(types.T_char) || to.Id != int32(types.T_varchar) || to.Width < from.Width {
+		return expr
+	}
+	from.Id, from.Width = to.Id, to.Width
+	if !sameFunctionalValueType(from, to) {
+		return expr
+	}
+	normalized := DeepCopyExpr(inner)
+	if !normalizeFunctionalExpression(normalized, tag) || !functionalJSONTextCastEligible(normalized, cols) {
+		return expr
+	}
+	return inner
+}

pkg/sql/plan/functional_json_index_test.go

@@ -0,0 +1,119 @@
+// Copyright 2026 Matrix Origin
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package plan
+
+import (
+	"testing"
+
+	"github.com/matrixorigin/matrixone/pkg/common/runtime"
+	"github.com/matrixorigin/matrixone/pkg/container/types"
+	"github.com/matrixorigin/matrixone/pkg/defines"
+	pb "github.com/matrixorigin/matrixone/pkg/pb/plan"
+	"github.com/matrixorigin/matrixone/pkg/sql/parsers"
+	"github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFunctionalJSONTextEligibility(t *testing.T) {
+	ctx := NewMockCompilerContext(false, newPlanTestProcess(t))
+	rt := runtime.ServiceRuntime(ctx.GetProcess().GetService())
+	old, had := rt.GetGlobalVariables(runtime.MOProtocolVersion)
+	rt.SetGlobalVariables(runtime.MOProtocolVersion, defines.MORPCLatestVersion)
+	t.Cleanup(func() {
+		if had {
+			rt.SetGlobalVariables(runtime.MOProtocolVersion, old)
+		} else {
+			rt.CompareAndDeleteGlobalVariables(runtime.MOProtocolVersion, defines.MORPCLatestVersion)
+		}
+	})
+	build := func(sql string) (*Plan, error) {
+		stmt, err := parsers.ParseOne(t.Context(), dialect.MYSQL, sql, 1)
+		if err != nil {
+			return nil, err
+		}
+		defer stmt.Free()
+		return BuildPlan(ctx, stmt, false)
+	}
+	for _, expression := range []string{
+		"cast(json_unquote(json_extract(doc,'$.kind')) as char(20))",
+		"cast(doc ->> '$.kind' as char(20))",
+		"cast(doc ->> '$[4294967295]' as char(20))",
+		"id + 1", "id - 1", "lower(name)",
+	} {
+		t.Run(expression, func(t *testing.T) {
+			p, err := build("create table docs(id int primary key,doc json,name varchar(20),index k((" + expression + ")))")
+			require.NoError(t, err)
+			table := p.GetDdl().GetCreateTable().TableDef
+			col := functionalIndexColumn(table, table.Indexes[0])
+			require.NotNil(t, col)
+			require.NotEqual(t, int32(types.T_json), col.Typ.Id, "JSON is never a physical index key")
+			require.NoError(t, validateFunctionalTable(t.Context(), table))
+		})
+	}
+	for _, expression := range []string{
+		"doc", "json_extract(doc,'$.kind')", "doc ->> '$.kind'",
+		"cast(doc ->> '$.*' as char(20))",
+		"cast(doc ->> '$[4294967296]' as char(20))",
+		"cast(doc ->> '$[999999999999999999999999999999999999999999]' as char(20))",
+		"cast(json_unquote(json_extract(doc,name)) as char(20))",
+		"cast(json_unquote(json_extract(doc,NULL)) as char(20))",
+		"cast(ts as char(20))",
+		"cast(json_unquote(json_extract(cast(ts as json),'$')) as char(20))",
+	} {
+		t.Run("reject/"+expression, func(t *testing.T) {
+			_, err := build("create table docs(id int primary key,doc json,name varchar(20),ts timestamp,index k((" + expression + ")))")
+			require.Error(t, err)
+		})
+	}
+	p, err := build("create table docs(id int primary key,doc json,index k((cast(doc ->> '$.kind' as char(20)))))")
+	require.NoError(t, err)
+	table := p.GetDdl().GetCreateTable().TableDef
+	key := functionalIndexColumn(table, table.Indexes[0]).GeneratedCol.Expr
+	badTable := DeepCopyTableDef(table, true)
+	badColumn := functionalIndexColumn(badTable, badTable.Indexes[0])
+	badColumn.GeneratedCol.Expr = DeepCopyExpr(key.GetF().Args[0].GetF().Args[0].GetF().Args[0])
+	badColumn.Typ = badColumn.GeneratedCol.Expr.Typ
+	require.Error(t, validateFunctionalTable(t.Context(), badTable), "restored metadata cannot introduce a JSON key")
+	for _, corrupt := range []func(*Expr){
+		func(e *Expr) { e.Typ.Id = int32(types.T_json) },
+		func(e *Expr) { e.GetF().Func.Obj++ },
+		func(e *Expr) { e.GetF().Args[0].GetF().Func.Obj++ },
+		func(e *Expr) { e.GetF().Args[0].GetF().Args[0].GetF().Func.Obj++ },
+		func(e *Expr) { e.GetF().Args[1].Typ.Width++ },
+		func(e *Expr) {
+			e.GetF().Args[0].GetF().Args[0].GetF().Args[1].GetLit().Value = &pb.Literal_Sval{Sval: "$[4294967296]"}
+		},
+	} {
+		bad := DeepCopyExpr(key)
+		corrupt(bad)
+		require.False(t, functionalExpressionEligible(bad, table.Cols, make(map[int32]bool)))
+	}
+	// Query binding must use the existing physical key while preserving the
+	// original expression as a base-row residual, rather than index JSON itself.
+	ctx.tables["docs"] = table
+	ctx.objects["docs"] = &ObjectRef{ObjName: "docs", SchemaName: "tpch"}
+	for _, indexTable := range p.GetDdl().GetCreateTable().IndexTables {
+		ctx.tables[indexTable.Name] = indexTable
+		ctx.objects[indexTable.Name] = &ObjectRef{ObjName: indexTable.Name, SchemaName: "tpch"}
+	}
+	query, err := build("select id from docs where cast(doc ->> '$.kind' as char(20))='alpha'")
+	require.NoError(t, err)
+	physical, residual := false, false
+	for _, node := range query.GetQuery().Nodes {
+		if node.NodeType != pb.Node_TABLE_SCAN || node.TableDef == nil {
+			continue
+		}
+		physical = physical || node.TableDef.Name == table.Indexes[0].IndexTableName
+		residual = residual || node.TableDef.Name == "docs" && len(node.FilterList) > 0
+	}
+	require.True(t, physical)
+	require.True(t, residual)
+}

test/distributed/cases/ddl/functional_json_key.result

@@ -0,0 +1,89 @@
+drop database if exists functional_json_key_bvt;
+create database functional_json_key_bvt;
+use functional_json_key_bvt;
+create table docs(id int primary key,doc json);
+insert into docs values(1,'{"kind":"alpha"}'),(2,'{"kind":"beta"}'),(3,'{}'),(4,'{"kind":null}'),(5,'{"kind":"xxxxxxxxxxxxxxxxxxxxx"}'),(6,NULL);
+create index idx_kind on docs ((cast(doc ->> '$.kind' as char(20))));
+create index idx_max_path on docs ((cast(doc ->> '$[4294967295]' as char(20))));
+select count(*) as hidden_generated from mo_catalog.mo_columns where att_database='functional_json_key_bvt' and att_relname='docs' and attr_has_generated=1;
+hidden_generated
+2
+drop index idx_max_path on docs;
+select count(*) as hidden_generated from mo_catalog.mo_columns where att_database='functional_json_key_bvt' and att_relname='docs' and attr_has_generated=1;
+hidden_generated
+1
+create table empty_docs(id int primary key,doc json);
+create index idx_json_value on empty_docs ((doc));
+not supported: functional index result type
+create index idx_json_extract on empty_docs ((json_extract(doc,'$.kind')));
+not supported: functional index expression must have session-independent semantics
+create index idx_unbounded_text on empty_docs ((doc ->> '$.kind'));
+not supported: functional index expression must have session-independent semantics
+create index idx_path_too_large on empty_docs ((cast(doc ->> '$[4294967296]' as char(20))));
+not supported: functional index expression must have session-independent semantics
+select count(*) as hidden_generated from mo_catalog.mo_columns where att_database='functional_json_key_bvt' and att_relname='empty_docs' and attr_has_generated=1;
+hidden_generated
+0
+create index idx_path_integer_overflow on empty_docs ((cast(doc ->> '$[999999999999999999999999999999999999]' as char(20))));
+not supported: functional index expression must have session-independent semantics
+select count(*) as hidden_generated from mo_catalog.mo_columns where att_database='functional_json_key_bvt' and att_relname='empty_docs' and attr_has_generated=1;
+hidden_generated
+0
+insert into empty_docs values(1,'{"kind":"alpha"}');
+select id from empty_docs where doc ->> '$.kind'='alpha';
+id
+1
+select count(*) as hidden_generated from mo_catalog.mo_columns where att_database='functional_json_key_bvt' and att_relname='empty_docs' and attr_has_generated=1;
+hidden_generated
+0
+select id from docs force index(idx_kind) where cast(doc ->> '$.kind' as char(20))='alpha' order by id;
+id
+1
+select id from docs ignore index(idx_kind) where cast(doc ->> '$.kind' as char(20))='alpha' order by id;
+id
+1
+select id from docs force index(idx_kind) where cast(doc ->> '$.kind' as char(20))='null' order by id;
+id
+4
+select id from docs force index(idx_kind) where cast(doc ->> '$.kind' as char(20))='xxxxxxxxxxxxxxxxxxxx' order by id;
+id
+5
+select id from docs where cast(doc ->> '$.kind' as char(20)) is null order by id;
+id
+3
+6
+update docs set doc='{"kind":"alpha"}' where id=2;
+select id from docs force index(idx_kind) where cast(doc ->> '$.kind' as char(20))='alpha' order by id;
+id
+1
+2
+begin;
+delete from docs where id=1;
+rollback;
+select id from docs force index(idx_kind) where cast(doc ->> '$.kind' as char(20))='alpha' order by id;
+id
+1
+2
+create table cloned like docs;
+insert into cloned select * from docs;
+select id from cloned force index(idx_kind) where cast(doc ->> '$.kind' as char(20))='alpha' order by id;
+id
+1
+2
+alter table docs add column extra int, algorithm=copy;
+select id from docs force index(idx_kind) where cast(doc ->> '$.kind' as char(20))='alpha' order by id;
+id
+1
+2
+select count(*) as hidden_generated from mo_catalog.mo_columns where att_database='functional_json_key_bvt' and att_relname='docs' and attr_has_generated=1;
+hidden_generated
+1
+drop index idx_kind on docs;
+select count(*) as hidden_generated from mo_catalog.mo_columns where att_database='functional_json_key_bvt' and att_relname='docs' and attr_has_generated=1;
+hidden_generated
+0
+select id from docs where cast(doc ->> '$.kind' as char(20))='alpha' order by id;
+id
+1
+2
+drop database functional_json_key_bvt;