Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
325 changes: 280 additions & 45 deletions internal/api/decode_body.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,16 @@ package api

import (
"bytes"
"cmp"
"encoding/json"
"errors"
"net/http"
"reflect"
"slices"
"strconv"
"strings"
"sync"
"unicode"
"unicode/utf8"
)

Expand All @@ -22,9 +27,12 @@ const maxDecodeWork = 256 << 10
// of the wrong JSON type. encoding/json keeps decoding after a type error
// but returns only the first one. unmarshalBody replaces the member that
// failed by null, which every type accepts as no change, and decodes again
// until the body decodes or a bound is reached. A body with one wrong-type
// field returns the error of encoding/json unchanged. Any other error
// (syntax, a wrong top-level type) is returned unchanged.
// until the body decodes or a bound is reached. It also stops when null
// does not fix the field (a map key that does not decode, an UnmarshalJSON
// that refuses null): when a pass leaves the body as it was, or the next
// decode names the same field with the same type again. A body with one
// wrong-type field returns the error of encoding/json unchanged. Any other
// error (syntax, a wrong top-level type) is returned unchanged.
func unmarshalBody(body []byte, dst any) error {
err := json.Unmarshal(body, dst)
te, ok := errors.AsType[*json.UnmarshalTypeError](err)
Expand All @@ -35,14 +43,15 @@ func unmarshalBody(body []byte, dst any) error {
errs.add(te)
work, budget := body, maxDecodeWork-len(body) // the first decode counts
for !errs.full() && budget >= len(work) {
next, ok := nullMembers(work, te.Field)
next, ok := nullMembers(work, reflect.TypeOf(dst), te)
// null is longer than a value of 1 to 3 bytes, so the body can grow.
if !ok || len(next) > budget {
if !ok || len(next) > budget || bytes.Equal(next, work) {
break
}
work, budget = next, budget-len(next)
prev := te
te, ok = errors.AsType[*json.UnmarshalTypeError](json.Unmarshal(work, dst))
if !ok || te.Field == "" {
if !ok || te.Field == "" || te.Field == prev.Field && te.Type == prev.Type {
break
}
errs.add(te)
Expand Down Expand Up @@ -149,60 +158,286 @@ func unknownFieldText(err error) string {
return "unknown field " + strconv.Quote(name[:cut]+"…") + " (" + strconv.Itoa(len(name)) + " bytes)"
}

// nullMembers returns body with every value at the dotted path field
// (members by name, array elements by index) replaced by null. It reports
// false when no value is found at the path.
func nullMembers(body []byte, field string) ([]byte, bool) {
spans := memberSpans(body, strings.Split(field, "."))
if len(spans) == 0 {
// nullMembers returns body with every value that the type error te names
// replaced by null; t is the type that body decodes into. It reports false
// when it finds no such value.
func nullMembers(body []byte, t reflect.Type, te *json.UnmarshalTypeError) ([]byte, bool) {
w := spanWalk{body: body, dec: json.NewDecoder(bytes.NewReader(body))}
if w.value(t, strings.Split(te.Field, ".")) != nil || len(w.spans) == 0 {
return nil, false
}
// A map key that holds "." gives two values one path (the keys "a.b"
// and "a" then "b"): the one that te names has the type it names. When
// none has it (te names a map key, or comes from an UnmarshalJSON),
// every value at the path is null.
if slices.ContainsFunc(w.spans, func(s valueSpan) bool { return s.typ == te.Type }) {
w.spans = slices.DeleteFunc(w.spans, func(s valueSpan) bool { return s.typ != te.Type })
Comment on lines +173 to +174

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve same-typed ambiguous path candidates

When ambiguous paths have the same recursive type, filtering solely by te.Type keeps and nulls multiple candidates, including values that have not failed yet. For example, with type R map[string]R, decoding {"r":{"a":{"b":1},"a.b":{"c":2}},"count":"x"} first reports r.a.b; both the nested value and the whole "a.b" map have type R, so this code nulls both and permanently hides the later r.a.b.c error. The resulting response therefore omits an invalid field despite unmarshalBody promising to name every wrong-type field.

Useful? React with 👍 / 👎.

}
var out bytes.Buffer
prev := 0
for _, sp := range spans {
out.Write(body[prev:sp[0]])
for _, sp := range w.spans {
out.Write(body[prev:sp.start])
out.WriteString("null")
prev = sp[1]
prev = sp.end
}
out.Write(body[prev:])
return out.Bytes(), true
}

// memberSpans returns the byte ranges of the values at path in the JSON
// value b. encoding/json matches a member name to a field without regard to
// case, and so does this.
func memberSpans(b []byte, path []string) [][2]int {
dec := json.NewDecoder(bytes.NewReader(b))
open, err := dec.Token()
if err != nil {
return nil
}
var spans [][2]int
for index := 0; dec.More(); index++ {
name := strconv.Itoa(index)
if open == json.Delim('{') {
key, err := dec.Token()
if err != nil {
return nil
// valueSpan is the byte range of a value in a body, and the type (without
// pointers) that the value decodes into.
type valueSpan struct {
start, end int
typ reflect.Type
}

// spanWalk finds the values at a path in one pass over a body, following
// the Go type that the body decodes into. The path is an
// UnmarshalTypeError.Field split at each ".": encoding/json spells the path
// of a value as its JSON Pointer (RFC 6901) with each "/" written as ".", so
// a member is its name with "~" as "~0" and "/" as "~1", and an array
// element is its index. A name takes one segment more than it holds ".". A
// member of a struct is the field that encoding/json decodes it into (see
// fieldFor), and members that differ in case can be one field; a member
// that is no field is on the path only as a key of the struct's `,embed`
// map (see bodyFields). A member of a map is its exact key.
// A type with its own UnmarshalJSON is walked as its fields, which is right
// for a method that decodes into them (wrefCreateBranchRequest).
type spanWalk struct {
body []byte
dec *json.Decoder
spans []valueSpan
}

// value reads the next value of the body, which decodes into t, and adds
// the span of each value at path in it.
func (w *spanWalk) value(t reflect.Type, path []string) error {
for t.Kind() == reflect.Pointer {
t = t.Elem()
}
// The value starts after the white space, ":" or "," that follow the
// last token.
start := len(w.body) - len(bytes.TrimLeft(w.body[w.dec.InputOffset():], " \t\r\n:,"))
if len(path) == 0 {
err := w.skip()
w.spans = append(w.spans, valueSpan{start, int(w.dec.InputOffset()), t})
return err
}
switch k := t.Kind(); {
case start == len(w.body):
case w.body[start] == '{' && (k == reflect.Struct || k == reflect.Map):
return w.members(t, path)
case w.body[start] == '[' && (k == reflect.Slice || k == reflect.Array):
return w.elements(t.Elem(), path)
}
return w.skip()
}

// members reads an object that decodes into the struct or map t.
func (w *spanWalk) members(t reflect.Type, path []string) error {
st := structFields{rest: t} // a map takes every member
if t.Kind() == reflect.Struct {
st = bodyFields(t)
}
if _, err := w.dec.Token(); err != nil {
return err
}
for w.dec.More() {
tok, err := w.dec.Token()
if err != nil {
return err
}
name, _ := tok.(string)
n := strings.Count(name, ".") + 1
var next reflect.Type // nil: the member is not on the path
if n <= len(path) {
seg := strings.Join(path[:n], ".")
if i := fieldFor(st.fields, name); i >= 0 {
if i == fieldFor(st.fields, strings.ReplaceAll(strings.ReplaceAll(seg, "~1", "/"), "~0", "~")) {
next = st.fields[i].typ
}
} else if st.rest != nil && st.rest.Kind() == reflect.Map && strings.ReplaceAll(strings.ReplaceAll(name, "~", "~0"), "/", "~1") == seg {
next = st.rest.Elem()
}
name, _ = key.(string)
}
var raw json.RawMessage
if dec.Decode(&raw) != nil {
return nil
if next == nil {
err = w.skip()
} else {
err = w.value(next, path[n:])
}
end := int(dec.InputOffset())
start := end - len(raw)
if !strings.EqualFold(name, path[0]) {
continue
if err != nil {
return err
}
if len(path) == 1 {
spans = append(spans, [2]int{start, end})
continue
}
_, err := w.dec.Token()
return err
}

// elements reads an array whose elements decode into t.
func (w *spanWalk) elements(t reflect.Type, path []string) error {
if _, err := w.dec.Token(); err != nil {
return err
}
for i := 0; w.dec.More(); i++ {
var err error
if strconv.Itoa(i) == path[0] {
err = w.value(t, path[1:])
} else {
err = w.skip()
}
for _, sp := range memberSpans(raw, path[1:]) {
spans = append(spans, [2]int{start + sp[0], start + sp[1]})
if err != nil {
return err
}
}
return spans
_, err := w.dec.Token()
return err
}

// skip reads the next value of the body.
func (w *spanWalk) skip() error {
var raw json.RawMessage
return w.dec.Decode(&raw)
}

// bodyField is a field of a struct type that encoding/json decodes: its
// JSON name, its type, its depth (more than 0 for a field of an embedded
// struct), whether its json tag gives the name, and whether the name
// matches in its own case only (the tag option case:strict).
type bodyField struct {
name string
typ reflect.Type
depth int
tagged, strict bool
}

// structFields is what bodyFields returns for a struct type.
type structFields struct {
fields []bodyField
rest reflect.Type // the `,embed` map or jsontext.Value that takes the members that no field takes, or nil
}

// bodyFields returns the fields of the struct type t that encoding/json
// decodes, in breadth-first order, by the rules of encoding/json (on
// json/v2, as Go 1.27 builds it): a struct or a pointer to one that is
// embedded without a JSON name, or has the option embed and no name, adds
// its fields one level deeper, and of the fields with one name only the
// shallowest one counts, or at one depth the only one whose json tag gives
// the name. Of the fields that take the members that no field takes (see
// takesRest), the shallowest one does, unless two share its depth.
func bodyFields(t reflect.Type) structFields {
if fields, ok := bodyFieldsCache.Load(t); ok {
return fields.(structFields)
}
type embedded struct {
t reflect.Type
depth int
follow bool // false for a type seen before: its embedded structs are not followed
}
queue := []embedded{{t, 0, true}}
seen := map[reflect.Type]bool{t: true}
var all, rests []bodyField
for i := 0; i < len(queue); i++ {
q := queue[i]
for j := range q.t.NumField() {
sf := q.t.Field(j)
tag := sf.Tag.Get("json")
if tag == "-" || !sf.IsExported() && !sf.Anonymous {
continue
}
name, embed, strict := jsonTag(tag)
ft := sf.Type
if ft.Kind() == reflect.Pointer && ft.Name() == "" {
ft = ft.Elem()
}
embed = name == "" && (embed || sf.Anonymous && ft.Kind() == reflect.Struct)
switch {
case embed && ft.Kind() == reflect.Struct:
if q.follow {
queue = append(queue, embedded{ft, q.depth + 1, !seen[ft]})
}
seen[ft] = true
case embed && sf.IsExported() && takesRest(ft):
rests = append(rests, bodyField{typ: ft, depth: q.depth})
case sf.IsExported() || ft.Kind() == reflect.Struct:
all = append(all, bodyField{cmp.Or(name, sf.Name), sf.Type, q.depth, name != "", strict})
}
}
}
var fields structFields
for i, f := range all {
counts := true
for j, g := range all {
if j != i && g.name == f.name && (g.depth < f.depth || g.depth == f.depth && (g.tagged || !f.tagged)) {
counts = false
break
}
}
if counts {
fields.fields = append(fields.fields, f)
}
}
if len(rests) == 1 || len(rests) > 1 && rests[0].depth < rests[1].depth {
fields.rest = rests[0].typ
}
bodyFieldsCache.Store(t, fields)
return fields
}

// bodyFieldsCache holds the bodyFields of each struct type.
var bodyFieldsCache sync.Map

// takesRest reports whether encoding/json gives the members that no field
// takes to an exported field of type t (without pointers) that has the
// option embed and no name: a jsontext.Value takes them as they are, and so
// does a map whose key type is a string kind without marshal methods. Any
// other such field is a plain field.
func takesRest(t reflect.Type) bool {
if t.PkgPath() == "encoding/json/jsontext" && t.Name() == "Value" {
return true
}
return t.Kind() == reflect.Map && t.Key().Kind() == reflect.String && !slices.ContainsFunc(marshalMethods, func(m string) bool {
_, ok := reflect.PointerTo(t.Key()).MethodByName(m)
return ok
Comment on lines +398 to +400

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Check method signatures before rejecting embedded maps

When an embedded map's string-like key type merely declares one of these method names with an unrelated signature, MethodByName classifies it as a custom-marshaled key even though encoding/json only checks whether the type implements the corresponding interface. For example, a key type with func (K) MarshalJSON() does not implement json.Marshaler, so encoding/json still routes unmatched members into map[K]int, while takesRest rejects that map; a body whose first unmatched member has the wrong value type then cannot be found by spanWalk, and later invalid fields are omitted. Check interface-compatible signatures rather than method names alone.

Useful? React with 👍 / 👎.

})
}

// marshalMethods are the methods by which a type marshals or unmarshals
// itself in encoding/json.
var marshalMethods = []string{"MarshalJSON", "MarshalJSONTo", "AppendText", "MarshalText", "UnmarshalJSON", "UnmarshalJSONFrom", "UnmarshalText"}

// jsonTag returns the JSON name that a json struct tag gives, "" for none,
// and whether its options hold embed, and case:strict without case:ignore,
// as encoding/json reads the tag (parseFieldOptions of json/v2). The name
// ends at the first ",". When a backslash, a quote or a backtick comes
// first, the name is the run of letters, digits and "_" that starts the
// tag, and none when that run is empty or starts with a digit. Invalid
// UTF-8 in the name reads as U+FFFD, as in a member name.
func jsonTag(tag string) (name string, embed, strict bool) {
name, opts, _ := strings.Cut(tag, ",")
if strings.ContainsAny(name, "\\'\"`") {
name = name[:len(name)-len(strings.TrimLeftFunc(name, func(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsNumber(r)
}))]
if r, _ := utf8.DecodeRuneInString(name); unicode.IsNumber(r) {
name = ""
}
}
ignore := false
for opt := range strings.SplitSeq(opts, ",") {
embed = embed || opt == "embed"
strict = strict || opt == "case:strict"
ignore = ignore || opt == "case:ignore"
}
return string([]rune(name)), embed, strict && !ignore
}

// fieldFor returns the index in fields of the field that encoding/json
// decodes a member name into: the field of that name, else the first one
// whose name matches it without regard to case and has no case:strict. It
// returns -1 when no field takes the name.
func fieldFor(fields []bodyField, name string) int {
if i := slices.IndexFunc(fields, func(f bodyField) bool { return f.name == name }); i >= 0 {
return i
}
return slices.IndexFunc(fields, func(f bodyField) bool { return !f.strict && strings.EqualFold(f.name, name) })
}
Loading
Loading