From 6ea053beafcdb35ab08e63d503aef74701d77a32 Mon Sep 17 00:00:00 2001 From: Alexander Wang Date: Sat, 12 Sep 2026 19:51:51 -0700 Subject: [PATCH 1/2] security: bound recursive glob expansion work --- d2compiler/compile.go | 5 + d2compiler/glob_expansion_test.go | 17 +++ d2ir/compile.go | 61 +++++++-- d2ir/d2ir.go | 44 ++++-- d2ir/expansion.go | 3 + d2ir/glob_expansion.go | 135 +++++++++++++++++++ d2ir/glob_expansion_test.go | 214 ++++++++++++++++++++++++++++++ d2ir/index_test.go | 16 +++ d2ir/pattern.go | 75 +++++++---- d2lib/d2.go | 5 + d2lib/d2_test.go | 9 ++ 11 files changed, 539 insertions(+), 45 deletions(-) create mode 100644 d2compiler/glob_expansion_test.go create mode 100644 d2ir/glob_expansion.go create mode 100644 d2ir/glob_expansion_test.go diff --git a/d2compiler/compile.go b/d2compiler/compile.go index 6734c1fb9c..465585ddf5 100644 --- a/d2compiler/compile.go +++ b/d2compiler/compile.go @@ -34,6 +34,10 @@ type CompileOptions struct { // MaxVariableExpansion bounds work added by variable substitutions and the // automatic copies they induce. Zero uses d2ir.DefaultMaxVariableExpansion. MaxVariableExpansion int64 + // MaxGlobExpansion bounds work performed by glob matching and + // materialization. Zero uses d2ir.DefaultMaxGlobExpansion. Explicit source + // fields are not counted as materialization work. + MaxGlobExpansion int64 // FS is the file system used for resolving imports in the D2 text. Nil // disables imports. Callers that accept untrusted input should prefer a // filesystem constrained to the intended import root; lib/localfile provides @@ -57,6 +61,7 @@ func Compile(p string, r io.Reader, opts *CompileOptions) (*d2graph.Graph, *d2ta Context: opts.Context, UTF16Pos: opts.UTF16Pos, MaxVariableExpansion: opts.MaxVariableExpansion, + MaxGlobExpansion: opts.MaxGlobExpansion, FS: opts.FS, }) if err != nil { diff --git a/d2compiler/glob_expansion_test.go b/d2compiler/glob_expansion_test.go new file mode 100644 index 0000000000..5bec430304 --- /dev/null +++ b/d2compiler/glob_expansion_test.go @@ -0,0 +1,17 @@ +package d2compiler_test + +import ( + "strings" + "testing" + + "github.com/d2lang/d2/d2compiler" +) + +func TestCompilePropagatesGlobExpansionLimit(t *testing.T) { + _, _, err := d2compiler.Compile("glob-feedback.d2", strings.NewReader("**.a\n**.b\n**.c\nx\n"), &d2compiler.CompileOptions{ + MaxGlobExpansion: 64, + }) + if err == nil || !strings.Contains(err.Error(), "glob expansion exceeds limit of 64 work units") { + t.Fatalf("Compile() error = %v, want glob expansion limit", err) + } +} diff --git a/d2ir/compile.go b/d2ir/compile.go index 2938aebfbc..391c73d3b8 100644 --- a/d2ir/compile.go +++ b/d2ir/compile.go @@ -35,8 +35,10 @@ type compiler struct { ctx context.Context contextErr error expansionErr error + globExpansionErr error halted bool variableExpansion *variableExpansionBudget + globExpansion *globExpansionBudget fs fs.FS imports []string @@ -81,6 +83,10 @@ type CompileOptions struct { // MaxVariableExpansion bounds work added by substitutions and automatic // copies. Zero uses DefaultMaxVariableExpansion. MaxVariableExpansion int64 + // MaxGlobExpansion bounds work performed by glob matching and + // materialization. Zero uses DefaultMaxGlobExpansion. Explicit source fields + // are not counted as materialization work. + MaxGlobExpansion int64 // FS resolves imports. Nil disables imports. The lib/localfile package // provides rooted and explicit unrestricted host-filesystem policies. FS fs.FS @@ -105,11 +111,16 @@ func Compile(ast *d2ast.Map, opts *CompileOptions) (*Map, []string, error) { if err != nil { return nil, nil, err } + globExpansion, err := newGlobExpansionBudget(opts.MaxGlobExpansion) + if err != nil { + return nil, nil, err + } c := &compiler{ err: &d2parser.ParseError{}, ctx: ctx, fs: opts.FS, variableExpansion: variableExpansion, + globExpansion: globExpansion, seenImports: make(map[string]struct{}), parsedImports: make(map[string]*d2ast.Map), @@ -132,25 +143,22 @@ func Compile(ast *d2ast.Map, opts *CompileOptions) (*Map, []string, error) { if c.contextErr != nil { return nil, nil, c.contextErr } + if err := c.compileLimitError(); err != nil { + return nil, nil, err + } c.compileSubstitutions(m, nil) if c.contextErr != nil { return nil, nil, c.contextErr } - if c.expansionErr != nil { - if !c.err.Empty() { - return nil, nil, c.err - } - return nil, nil, c.expansionErr + if err := c.compileLimitError(); err != nil { + return nil, nil, err } c.overlayClasses(m) if c.contextErr != nil { return nil, nil, c.contextErr } - if c.expansionErr != nil { - if !c.err.Empty() { - return nil, nil, c.err - } - return nil, nil, c.expansionErr + if err := c.compileLimitError(); err != nil { + return nil, nil, err } // Substitutions can grow shared nodes after an earlier alias inserted them // (for example through a forward scalar chain in an array spread). Recheck @@ -162,8 +170,8 @@ func Compile(ast *d2ast.Map, opts *CompileOptions) (*Map, []string, error) { if err := ctx.Err(); err != nil { return nil, nil, err } - if c.expansionErr != nil && c.err.Empty() { - return nil, nil, c.expansionErr + if err := c.compileLimitError(); err != nil { + return nil, nil, err } if !c.err.Empty() { return nil, nil, c.err @@ -171,6 +179,19 @@ func Compile(ast *d2ast.Map, opts *CompileOptions) (*Map, []string, error) { return m, c.imports, nil } +func (c *compiler) compileLimitError() error { + if c.expansionErr == nil && c.globExpansionErr == nil { + return nil + } + if !c.err.Empty() { + return c.err + } + if c.expansionErr != nil { + return c.expansionErr + } + return c.globExpansionErr +} + func (c *compiler) overlayClasses(m *Map) { if c.stopped() { return @@ -1001,6 +1022,11 @@ func (c *compiler) compileMap(dst *Map, ast, scopeAST *d2ast.Map) { }) case n.Substitution != nil: // placeholder field to be resolved at the end + if len(c.globRefContextStack) > 0 { + if !c.reserveGlobGeneratedFieldWork(dst, n.Substitution) || !c.reserveGlobField(n.Substitution) { + return + } + } f := &Field{ parent: dst, Primary_: &Scalar{ @@ -1117,6 +1143,11 @@ func (c *compiler) compileKey(refctx *RefContext) { return } postTargetStart := len(c.lazyPostTargets) + if refctx.Key.HasGlob() || len(c.globRefContextStack) > 0 { + if !c.reserveGlobWork(refctx.Key, 1) { + return + } + } if refctx.Key.HasGlob() { for _, refctx2 := range c.globRefContextStack { if refctx.Equal(refctx2) { @@ -2239,8 +2270,12 @@ func (c *compiler) compileArray(dst *Array, a *d2ast.Array, scopeAST *d2ast.Map) if c.stopped() { return } + arrayNode := an.Unbox() + if len(c.globRefContextStack) > 0 && !c.reserveGlobWork(arrayNode, 1) { + return + } var irv Value - switch v := an.Unbox().(type) { + switch v := arrayNode.(type) { case *d2ast.Array: ira := &Array{ parent: dst, diff --git a/d2ir/d2ir.go b/d2ir/d2ir.go index f2ea876ced..de690cad34 100644 --- a/d2ir/d2ir.go +++ b/d2ir/d2ir.go @@ -1214,6 +1214,12 @@ func (m *Map) ensureFieldMode(kp *d2ast.KeyPath, refctx *RefContext, create bool } func (m *Map) ensureField(i int, kp *d2ast.KeyPath, refctx *RefContext, create bool, gctx *globContext, c *compiler, indexed bool, fa, created *[]*Field) error { + visitGlobCandidate := func(*Field) bool { + if c == nil || gctx == nil { + return true + } + return c.reserveGlobWork(c.globSource(refctx), 1) + } filter := func(f *Field, passthrough bool) bool { if gctx != nil { var ks string @@ -1255,10 +1261,13 @@ func (m *Map) ensureField(i int, kp *d2ast.KeyPath, refctx *RefContext, create b var multi bool if c != nil && c.lazyGlobTarget != nil && gctx != nil && (d2ast.IsDoubleGlob(us.Pattern) || d2ast.IsTripleGlob(us.Pattern)) { - fa2 = m.multiGlobMatchesToward(c.lazyGlobTarget, us.Pattern) + fa2 = m.multiGlobMatchesToward(c.lazyGlobTarget, us.Pattern, visitGlobCandidate) multi = true } else { - fa2, multi = m.multiGlob(us.Pattern) + fa2, multi = m.multiGlob(us.Pattern, visitGlobCandidate) + } + if c != nil && c.stopped() { + return nil } if multi { if i == len(kp.Path)-1 { @@ -1290,6 +1299,9 @@ func (m *Map) ensureField(i int, kp *d2ast.KeyPath, refctx *RefContext, create b } } for _, f := range fields { + if !visitGlobCandidate(f) { + return nil + } if f.Name == nil { continue } @@ -1346,12 +1358,14 @@ func (m *Map) ensureField(i int, kp *d2ast.KeyPath, refctx *RefContext, create b if f := existing; f != nil { // Don't add references for fake common KeyPath from trimCommon in CreateEdge. if refctx != nil { + dueToGlob := c != nil && len(c.globRefContextStack) > 0 + dueToLazyGlob := c != nil && c.lazyGlobBeingApplied f.appendReference(&FieldReference{ String: kp.Path[i].Unbox(), KeyPath: kp, Context_: refctx, - DueToGlob_: len(c.globRefContextStack) > 0, - DueToLazyGlob_: c.lazyGlobBeingApplied, + DueToGlob_: dueToGlob, + DueToLazyGlob_: dueToLazyGlob, }) } @@ -1376,18 +1390,25 @@ func (m *Map) ensureField(i int, kp *d2ast.KeyPath, refctx *RefContext, create b if !create { return nil } - if _, ok := d2ast.ReservedKeywords[strings.ToLower(head.ScalarString())]; !(ok && head.IsUnquoted()) && len(c.globRefContextStack) > 0 { + if _, ok := d2ast.ReservedKeywords[strings.ToLower(head.ScalarString())]; !(ok && head.IsUnquoted()) && c != nil && len(c.globRefContextStack) > 0 { shape := ParentShape(m) if shape == d2target.ShapeClass || shape == d2target.ShapeSQLTable { return nil } } + var globSource d2ast.Node + if c != nil && len(c.globRefContextStack) > 0 { + globSource = c.globSource(refctx) + if !c.reserveGlobGeneratedFieldWork(m, globSource) { + return nil + } + } f := &Field{ parent: m, Name: kp.Path[i].Unbox(), } defer func() { - if i < kp.FirstGlob() { + if c == nil || i < kp.FirstGlob() { return } for _, grefctx := range c.globRefContextStack { @@ -1403,17 +1424,24 @@ func (m *Map) ensureField(i int, kp *d2ast.KeyPath, refctx *RefContext, create b }() // Don't add references for fake common KeyPath from trimCommon in CreateEdge. if refctx != nil { + dueToGlob := c != nil && len(c.globRefContextStack) > 0 + dueToLazyGlob := c != nil && c.lazyGlobBeingApplied f.appendReference(&FieldReference{ String: kp.Path[i].Unbox(), KeyPath: kp, Context_: refctx, - DueToGlob_: len(c.globRefContextStack) > 0, - DueToLazyGlob_: c.lazyGlobBeingApplied, + DueToGlob_: dueToGlob, + DueToLazyGlob_: dueToLazyGlob, }) } if !filter(f, true) { return nil } + if c != nil && len(c.globRefContextStack) > 0 { + if !c.reserveGlobField(globSource) { + return nil + } + } m.appendField(f) *created = append(*created, f) if i+1 == len(kp.Path) { diff --git a/d2ir/expansion.go b/d2ir/expansion.go index 65d8f6fda8..eeecc89777 100644 --- a/d2ir/expansion.go +++ b/d2ir/expansion.go @@ -63,6 +63,9 @@ func (c *compiler) stopped() bool { if c.variableExpansion == nil { c.variableExpansion = &variableExpansionBudget{limit: DefaultMaxVariableExpansion} } + if c.globExpansion == nil { + c.globExpansion = &globExpansionBudget{limit: DefaultMaxGlobExpansion} + } if err := c.ctx.Err(); err != nil { c.contextErr = err c.halted = true diff --git a/d2ir/glob_expansion.go b/d2ir/glob_expansion.go new file mode 100644 index 0000000000..4c662b2b1e --- /dev/null +++ b/d2ir/glob_expansion.go @@ -0,0 +1,135 @@ +package d2ir + +import ( + "fmt" + "math" + + "github.com/d2lang/d2/d2ast" +) + +// DefaultMaxGlobExpansion is the maximum amount of work globs may perform +// during one compilation. Explicit source fields do not consume this budget. +const DefaultMaxGlobExpansion int64 = 65_536 + +// maxGlobCreatedFields independently caps retained materialization. Work is +// normally the tighter bound for deep expansions, while this ceiling bounds +// broad, shallow expansions even when callers raise MaxGlobExpansion. +const maxGlobCreatedFields int64 = 10_000 + +type globExpansionBudget struct { + limit int64 + used int64 + createdFields int64 +} + +type globExpansionLimitError struct { + limit int64 +} + +func (e *globExpansionLimitError) Error() string { + return fmt.Sprintf("glob expansion exceeds limit of %d work units", e.limit) +} + +type globFieldLimitError struct { + limit int64 +} + +func (e *globFieldLimitError) Error() string { + return fmt.Sprintf("glob expansion exceeds limit of %d created fields", e.limit) +} + +func newGlobExpansionBudget(limit int64) (*globExpansionBudget, error) { + if limit < 0 { + return nil, fmt.Errorf("MaxGlobExpansion must not be negative") + } + if limit == 0 { + limit = DefaultMaxGlobExpansion + } + return &globExpansionBudget{limit: limit}, nil +} + +func (b *globExpansionBudget) reserve(units int64) error { + if units < 0 || units > b.limit-b.used { + return &globExpansionLimitError{limit: b.limit} + } + b.used += units + return nil +} + +func (b *globExpansionBudget) reserveField() error { + if b.createdFields >= maxGlobCreatedFields { + return &globFieldLimitError{limit: maxGlobCreatedFields} + } + b.createdFields++ + return nil +} + +func (c *compiler) globExpansionFailure(n d2ast.Node, err error) bool { + c.globExpansionErr = err + if n != nil { + c.errorf(n, "%v", err) + } + c.halted = true + return false +} + +func (c *compiler) reserveGlobWork(n d2ast.Node, units int64) bool { + if c.stopped() { + return false + } + if err := c.globExpansion.reserve(units); err != nil { + return c.globExpansionFailure(n, err) + } + return true +} + +func (c *compiler) reserveGlobField(n d2ast.Node) bool { + if c.stopped() { + return false + } + if err := c.globExpansion.reserveField(); err != nil { + return c.globExpansionFailure(n, err) + } + return true +} + +func (c *compiler) globSource(refctx *RefContext) d2ast.Node { + if refctx != nil && refctx.Key != nil { + return refctx.Key + } + if len(c.globRefContextStack) == 0 { + return nil + } + return c.globRefContextStack[len(c.globRefContextStack)-1].Key +} + +// reserveGlobGeneratedFieldWork accounts for the path formatting and applied +// set lookups performed before a glob can materialize a field. Charging the +// prospective depth once per active glob context makes alternating recursive +// rules consume budget before constructing ever-deeper IR paths. +func (c *compiler) reserveGlobGeneratedFieldWork(parent *Map, n d2ast.Node) bool { + if c.stopped() { + return false + } + active := int64(len(c.globRefContextStack)) + if active == 0 { + return true + } + + depth := int64(1) + for f := ParentField(parent); f != nil && !f.Root(); { + if depth == math.MaxInt64 { + return c.globExpansionFailure(n, &globExpansionLimitError{limit: c.globExpansion.limit}) + } + depth++ + parent = ParentMap(f) + if parent == nil { + break + } + f = ParentField(parent) + } + if depth > math.MaxInt64/active { + return c.globExpansionFailure(n, &globExpansionLimitError{limit: c.globExpansion.limit}) + } + return c.reserveGlobWork(n, depth*active) +} diff --git a/d2ir/glob_expansion_test.go b/d2ir/glob_expansion_test.go new file mode 100644 index 0000000000..304d0a3453 --- /dev/null +++ b/d2ir/glob_expansion_test.go @@ -0,0 +1,214 @@ +package d2ir + +import ( + "fmt" + "strings" + "testing" + + "github.com/d2lang/d2/d2parser" +) + +func TestGlobExpansionBudgetDefault(t *testing.T) { + budget, err := newGlobExpansionBudget(0) + if err != nil { + t.Fatal(err) + } + if budget.limit != DefaultMaxGlobExpansion { + t.Fatalf("zero-value limit = %d, want %d", budget.limit, DefaultMaxGlobExpansion) + } + for range DefaultMaxGlobExpansion { + if err := budget.reserve(1); err != nil { + t.Fatal(err) + } + } + if err := budget.reserve(1); err == nil { + t.Fatal("reservation beyond the default glob expansion limit succeeded") + } + + budget, err = newGlobExpansionBudget(0) + if err != nil { + t.Fatal(err) + } + for range maxGlobCreatedFields { + if err := budget.reserveField(); err != nil { + t.Fatal(err) + } + } + if err := budget.reserveField(); err == nil { + t.Fatal("reservation beyond the glob-created field limit succeeded") + } +} + +func TestGlobExpansionBudgetDoesNotCountExplicitFields(t *testing.T) { + var explicit strings.Builder + for i := range 100 { + fmt.Fprintf(&explicit, "field-%d\n", i) + } + ast, err := d2parser.Parse("explicit-fields.d2", strings.NewReader(explicit.String()), nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := Compile(ast, &CompileOptions{MaxGlobExpansion: 1}); err != nil { + t.Fatalf("explicit fields consumed the glob expansion budget: %v", err) + } + +} + +func TestNormalGlobExpansionFitsDefaultWorkBudget(t *testing.T) { + var source strings.Builder + source.WriteString("*.style.fill: red\n") + for i := range 2_000 { + fmt.Fprintf(&source, "field-%d\n", i) + } + ast, err := d2parser.Parse("normal-glob.d2", strings.NewReader(source.String()), nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := Compile(ast, nil); err != nil { + t.Fatal(err) + } +} + +func TestRecursiveGlobFeedbackIsBounded(t *testing.T) { + const source = "**.a\n**.b\n**.c\nx\n" + ast, err := d2parser.Parse("glob-feedback.d2", strings.NewReader(source), nil) + if err != nil { + t.Fatal(err) + } + _, _, err = Compile(ast, &CompileOptions{MaxGlobExpansion: 64}) + want := "glob expansion exceeds limit of 64 work units" + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("Compile() error = %v, want %q", err, want) + } + if !strings.Contains(err.Error(), "glob-feedback.d2:") { + t.Fatalf("Compile() error = %v, want source location", err) + } +} + +func TestTwoRuleRecursiveGlobFeedbackIsBoundedByDefault(t *testing.T) { + const source = "**.a\n**.b\nx\n" + ast, err := d2parser.Parse("two-rule-glob-feedback.d2", strings.NewReader(source), nil) + if err != nil { + t.Fatal(err) + } + _, _, err = Compile(ast, nil) + want := fmt.Sprintf("glob expansion exceeds limit of %d work units", DefaultMaxGlobExpansion) + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("Compile() error = %v, want %q", err, want) + } +} + +func TestGlobGeneratedSubstitutionPlaceholdersConsumeBudget(t *testing.T) { + const limit = 16 + const onePlaceholder = "vars: {x: {p}}\n*.a: { ...${x} }\nz\n" + ast, err := d2parser.Parse("one-glob-placeholder.d2", strings.NewReader(onePlaceholder), nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := Compile(ast, &CompileOptions{MaxGlobExpansion: limit}); err != nil { + t.Fatalf("one-placeholder boundary failed: %v", err) + } + + const twoPlaceholders = "vars: {x: {p}; y: {q}}\n*.a: { ...${x}; ...${y} }\nz\n" + ast, err = d2parser.Parse("two-glob-placeholders.d2", strings.NewReader(twoPlaceholders), nil) + if err != nil { + t.Fatal(err) + } + _, _, err = Compile(ast, &CompileOptions{MaxGlobExpansion: limit}) + if err == nil || !strings.Contains(err.Error(), "glob expansion exceeds limit of 16 work units") { + t.Fatalf("two-placeholder Compile() error = %v, want glob work limit", err) + } +} + +func TestTripleGlobSkipsUnresolvedSubstitutionPlaceholder(t *testing.T) { + const source = "...${missing}\n***.a\nx\n" + ast, err := d2parser.Parse("triple-glob-placeholder.d2", strings.NewReader(source), nil) + if err != nil { + t.Fatal(err) + } + _, _, err = Compile(ast, nil) + if err == nil || !strings.Contains(err.Error(), `could not resolve variable "missing"`) { + t.Fatalf("Compile() error = %v, want unresolved-variable error", err) + } +} + +func TestLiteralGlobBodyWorkConsumesBudget(t *testing.T) { + var source strings.Builder + source.WriteString("*.a: {") + for range 64 { + source.WriteString(" b: v;") + } + source.WriteString(" }\nz\n") + + ast, err := d2parser.Parse("literal-glob-body.d2", strings.NewReader(source.String()), nil) + if err != nil { + t.Fatal(err) + } + _, _, err = Compile(ast, &CompileOptions{MaxGlobExpansion: 64}) + if err == nil || !strings.Contains(err.Error(), "glob expansion exceeds limit of 64 work units") { + t.Fatalf("Compile() error = %v, want glob work limit", err) + } +} + +func TestGlobArrayNodeWorkConsumesBudget(t *testing.T) { + const array = "[[${x}]]" + normalSource := "vars: {x: value}\nitems: " + array + "\n" + ast, err := d2parser.Parse("normal-array.d2", strings.NewReader(normalSource), nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := Compile(ast, &CompileOptions{MaxGlobExpansion: 1}); err != nil { + t.Fatalf("normal array consumed glob work budget: %v", err) + } + + globSource := "vars: {x: value}\n*.items: " + array + "\nz\n" + for _, tc := range []struct { + limit int64 + location string + }{ + {limit: 6, location: "glob-array.d2:2:11"}, + {limit: 7, location: "glob-array.d2:2:12"}, + } { + ast, err = d2parser.Parse("glob-array.d2", strings.NewReader(globSource), nil) + if err != nil { + t.Fatal(err) + } + _, _, err = Compile(ast, &CompileOptions{MaxGlobExpansion: tc.limit}) + want := fmt.Sprintf("glob expansion exceeds limit of %d work units", tc.limit) + if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), tc.location) { + t.Fatalf("Compile() error = %v, want %q at %s", err, want, tc.location) + } + } + + ast, err = d2parser.Parse("glob-array.d2", strings.NewReader(globSource), nil) + if err != nil { + t.Fatal(err) + } + if _, _, err := Compile(ast, &CompileOptions{MaxGlobExpansion: 12}); err != nil { + t.Fatalf("exact glob-array work boundary failed: %v", err) + } +} + +func TestGlobExpansionRejectsNegativeLimit(t *testing.T) { + ast, err := d2parser.Parse("glob-negative.d2", strings.NewReader("x"), nil) + if err != nil { + t.Fatal(err) + } + _, _, err = Compile(ast, &CompileOptions{MaxGlobExpansion: -1}) + if err == nil || err.Error() != "MaxGlobExpansion must not be negative" { + t.Fatalf("Compile() error = %v, want negative-limit error", err) + } +} + +func TestDefaultRecursiveGlobFeedbackLimitMessage(t *testing.T) { + const source = "**.a\n**.b\n**.c\nx\n" + ast, err := d2parser.Parse("glob-feedback-default.d2", strings.NewReader(source), nil) + if err != nil { + t.Fatal(err) + } + _, _, err = Compile(ast, nil) + want := fmt.Sprintf("glob expansion exceeds limit of %d work units", DefaultMaxGlobExpansion) + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("Compile() error = %v, want %q", err, want) + } +} diff --git a/d2ir/index_test.go b/d2ir/index_test.go index ff222160a1..a2898c5f08 100644 --- a/d2ir/index_test.go +++ b/d2ir/index_test.go @@ -95,6 +95,22 @@ func TestPublicMapLookupsRemainSafeAfterDirectMutation(t *testing.T) { wg.Wait() } +func TestEnsureFieldCreateWithoutCompiler(t *testing.T) { + m := &Map{} + m.initRoot() + key, err := d2parser.ParseKey("created") + if err != nil { + t.Fatal(err) + } + fields, err := m.EnsureField(key, nil, true, nil) + if err != nil { + t.Fatal(err) + } + if len(fields) != 1 || fields[0].Name.ScalarString() != "created" { + t.Fatalf("EnsureField() = %#v, want newly created field", fields) + } +} + func compileIndexTestSource(t *testing.T, source string) *Map { t.Helper() ast, err := d2parser.Parse("index-test.d2", strings.NewReader(source), nil) diff --git a/d2ir/pattern.go b/d2ir/pattern.go index 6ff2f52ed5..e469f17e1d 100644 --- a/d2ir/pattern.go +++ b/d2ir/pattern.go @@ -6,14 +6,14 @@ import ( "github.com/d2lang/d2/d2ast" ) -func (m *Map) multiGlob(pattern []string) ([]*Field, bool) { +func (m *Map) multiGlob(pattern []string, visit func(*Field) bool) ([]*Field, bool) { var fa []*Field if d2ast.IsDoubleGlob(pattern) { - m._doubleGlob(&fa) + m._doubleGlob(&fa, visit) return fa, true } if d2ast.IsTripleGlob(pattern) { - m._tripleGlob(&fa) + m._tripleGlob(&fa, visit) return fa, true } return nil, false @@ -53,43 +53,47 @@ func (m *Map) directChildToward(target *Field) *Field { // toward target that the existing recursive glob traversal would append. A // whole branch is replayed (rather than only the target path) so reference and // filter ordering remains identical within the affected branch. -func (m *Map) multiGlobMatchesToward(target *Field, pattern []string) []*Field { +func (m *Map) multiGlobMatchesToward(target *Field, pattern []string, visit func(*Field) bool) []*Field { path := m.pathToField(target) if len(path) == 0 { return nil } var matches []*Field if d2ast.IsDoubleGlob(pattern) { - _doubleGlobField(path[0], &matches) + _doubleGlobField(path[0], &matches, visit) return matches } if d2ast.IsTripleGlob(pattern) { - _tripleGlobField(path[0], &matches) + _tripleGlobField(path[0], &matches, visit) return matches } return nil } -func _doubleGlobField(f *Field, matches *[]*Field) { +func _doubleGlobField(f *Field, matches *[]*Field, visit func(*Field) bool) bool { if f == nil || f.Name == nil { - return + return true + } + if visit != nil && !visit(f) { + return false } name := f.Name.ScalarString() if _, reserved := d2ast.ReservedKeywords[name]; reserved && f.Name.IsUnquoted() { if skipDoubleGlobSubtree(name) { - return + return true } if f.Map() != nil { - f.Map()._doubleGlob(matches) + return f.Map()._doubleGlob(matches, visit) } - return + return true } if NodeBoardKind(f) == "" { *matches = append(*matches, f) } if f.Map() != nil { - f.Map()._doubleGlob(matches) + return f.Map()._doubleGlob(matches, visit) } + return true } func skipDoubleGlobSubtree(name string) bool { @@ -101,30 +105,37 @@ func skipDoubleGlobSubtree(name string) bool { return name == "classes" || name == "vars" } -func _tripleGlobField(f *Field, matches *[]*Field) { +func _tripleGlobField(f *Field, matches *[]*Field, visit func(*Field) bool) bool { if f == nil || f.Name == nil { - return + return true + } + if visit != nil && !visit(f) { + return false } name := f.Name.ScalarString() if _, reserved := d2ast.ReservedKeywords[name]; reserved && f.Name.IsUnquoted() { if _, board := d2ast.BoardKeywords[name]; !board { - return + return true } if f.Map() != nil { - f.Map()._tripleGlob(matches) + return f.Map()._tripleGlob(matches, visit) } - return + return true } if NodeBoardKind(f) == "" { *matches = append(*matches, f) } if f.Map() != nil { - f.Map()._tripleGlob(matches) + return f.Map()._tripleGlob(matches, visit) } + return true } -func (m *Map) _doubleGlob(fa *[]*Field) { +func (m *Map) _doubleGlob(fa *[]*Field, visit func(*Field) bool) bool { for _, f := range m.Fields { + if visit != nil && !visit(f) { + return false + } if f.Name == nil { continue } @@ -134,7 +145,9 @@ func (m *Map) _doubleGlob(fa *[]*Field) { continue } if f.Map() != nil { - f.Map()._doubleGlob(fa) + if !f.Map()._doubleGlob(fa, visit) { + return false + } } continue } @@ -142,20 +155,31 @@ func (m *Map) _doubleGlob(fa *[]*Field) { *fa = append(*fa, f) } if f.Map() != nil { - f.Map()._doubleGlob(fa) + if !f.Map()._doubleGlob(fa, visit) { + return false + } } } + return true } -func (m *Map) _tripleGlob(fa *[]*Field) { +func (m *Map) _tripleGlob(fa *[]*Field, visit func(*Field) bool) bool { for _, f := range m.Fields { + if visit != nil && !visit(f) { + return false + } + if f == nil || f.Name == nil { + continue + } if _, ok := d2ast.ReservedKeywords[f.Name.ScalarString()]; ok && f.Name.IsUnquoted() { if _, ok := d2ast.BoardKeywords[f.Name.ScalarString()]; !ok { continue } // We don't ever want to append layers, scenarios or steps directly. if f.Map() != nil { - f.Map()._tripleGlob(fa) + if !f.Map()._tripleGlob(fa, visit) { + return false + } } continue } @@ -163,9 +187,12 @@ func (m *Map) _tripleGlob(fa *[]*Field) { *fa = append(*fa, f) } if f.Map() != nil { - f.Map()._tripleGlob(fa) + if !f.Map()._tripleGlob(fa, visit) { + return false + } } } + return true } func matchPattern(s string, pattern []string) bool { diff --git a/d2lib/d2.go b/d2lib/d2.go index 8ce530c95b..1153f0c8f9 100644 --- a/d2lib/d2.go +++ b/d2lib/d2.go @@ -26,6 +26,10 @@ type CompileOptions struct { // MaxVariableExpansion bounds work added by variable substitutions and the // automatic copies they induce. Zero uses the secure compiler default. MaxVariableExpansion int64 + // MaxGlobExpansion bounds work performed by glob matching and + // materialization. Zero uses the secure compiler default. Explicit source + // fields are not counted as materialization work. + MaxGlobExpansion int64 // FS is the file system used for resolving imports in the D2 text. Nil // disables imports. Callers that accept untrusted input should prefer a // filesystem constrained to the intended import root; lib/localfile provides @@ -85,6 +89,7 @@ func compileInput(ctx context.Context, input string, compileOpts *CompileOptions Context: ctx, UTF16Pos: compileOpts.UTF16Pos, MaxVariableExpansion: compileOpts.MaxVariableExpansion, + MaxGlobExpansion: compileOpts.MaxGlobExpansion, FS: compileOpts.FS, }) if err != nil { diff --git a/d2lib/d2_test.go b/d2lib/d2_test.go index 01d79fa1ba..285a047cfe 100644 --- a/d2lib/d2_test.go +++ b/d2lib/d2_test.go @@ -37,6 +37,15 @@ func TestCompilePropagatesVariableExpansionLimit(t *testing.T) { } } +func TestCompilePropagatesGlobExpansionLimit(t *testing.T) { + _, _, err := Compile(context.Background(), "**.a\n**.b\n**.c\nx\n", &CompileOptions{ + MaxGlobExpansion: 64, + }, nil) + if err == nil || !strings.Contains(err.Error(), "glob expansion exceeds limit of 64 work units") { + t.Fatalf("Compile() error = %v, want glob expansion limit", err) + } +} + func TestNilFSDeniesImports(t *testing.T) { directory := t.TempDir() if err := os.WriteFile(filepath.Join(directory, "secret.d2"), []byte("disclosed"), 0o600); err != nil { From 4c6c480578122e6cd2335460cb5c0b2b7d4e7a1f Mon Sep 17 00:00:00 2001 From: Alexander Wang Date: Sat, 12 Sep 2026 20:21:27 -0700 Subject: [PATCH 2/2] docs: note recursive glob limits --- ci/release/changelogs/next.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/release/changelogs/next.md b/ci/release/changelogs/next.md index 33c9700fbc..9b56387866 100644 --- a/ci/release/changelogs/next.md +++ b/ci/release/changelogs/next.md @@ -42,6 +42,9 @@ configure the limit. [#2923](https://github.com/d2lang/d2/pull/2923) - Honor cancellation while expanding substitutions and materializing compiled graphs. [#2923](https://github.com/d2lang/d2/pull/2923) + - Limit recursive glob matching and generated-field work so small diagrams + fail with a clear error instead of consuming excessive CPU or memory. + [#2925](https://github.com/d2lang/d2/pull/2925) - decoding and assets: - Cap decompressed URL-encoded D2 input at 16 MiB. [#2902](https://github.com/d2lang/d2/pull/2902) - Bound image references, locators, fetched and decoded bytes, cached data, and