Repository navigation
fix(api): find map keys with dots, slashes or tildes in unmarshalBody #883
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
bb143a2
1fc0dbb
b7de6f2
ac74193
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,11 +2,16 @@ package api | |
|
|
||
| import ( | ||
| "bytes" | ||
| "cmp" | ||
| "encoding/json" | ||
| "errors" | ||
| "net/http" | ||
| "reflect" | ||
| "slices" | ||
| "strconv" | ||
| "strings" | ||
| "sync" | ||
| "unicode" | ||
| "unicode/utf8" | ||
| ) | ||
|
|
||
|
|
@@ -22,9 +27,12 @@ const maxDecodeWork = 256 << 10 | |
| // of the wrong JSON type. encoding/json keeps decoding after a type error | ||
| // but returns only the first one. unmarshalBody replaces the member that | ||
| // failed by null, which every type accepts as no change, and decodes again | ||
| // until the body decodes or a bound is reached. A body with one wrong-type | ||
| // field returns the error of encoding/json unchanged. Any other error | ||
| // (syntax, a wrong top-level type) is returned unchanged. | ||
| // until the body decodes or a bound is reached. It also stops when null | ||
| // does not fix the field (a map key that does not decode, an UnmarshalJSON | ||
| // that refuses null): when a pass leaves the body as it was, or the next | ||
| // decode names the same field with the same type again. A body with one | ||
| // wrong-type field returns the error of encoding/json unchanged. Any other | ||
| // error (syntax, a wrong top-level type) is returned unchanged. | ||
| func unmarshalBody(body []byte, dst any) error { | ||
| err := json.Unmarshal(body, dst) | ||
| te, ok := errors.AsType[*json.UnmarshalTypeError](err) | ||
|
|
@@ -35,14 +43,15 @@ func unmarshalBody(body []byte, dst any) error { | |
| errs.add(te) | ||
| work, budget := body, maxDecodeWork-len(body) // the first decode counts | ||
| for !errs.full() && budget >= len(work) { | ||
| next, ok := nullMembers(work, te.Field) | ||
| next, ok := nullMembers(work, reflect.TypeOf(dst), te) | ||
| // null is longer than a value of 1 to 3 bytes, so the body can grow. | ||
| if !ok || len(next) > budget { | ||
| if !ok || len(next) > budget || bytes.Equal(next, work) { | ||
| break | ||
| } | ||
| work, budget = next, budget-len(next) | ||
| prev := te | ||
| te, ok = errors.AsType[*json.UnmarshalTypeError](json.Unmarshal(work, dst)) | ||
| if !ok || te.Field == "" { | ||
| if !ok || te.Field == "" || te.Field == prev.Field && te.Type == prev.Type { | ||
| break | ||
| } | ||
| errs.add(te) | ||
|
|
@@ -149,60 +158,286 @@ func unknownFieldText(err error) string { | |
| return "unknown field " + strconv.Quote(name[:cut]+"…") + " (" + strconv.Itoa(len(name)) + " bytes)" | ||
| } | ||
|
|
||
| // nullMembers returns body with every value at the dotted path field | ||
| // (members by name, array elements by index) replaced by null. It reports | ||
| // false when no value is found at the path. | ||
| func nullMembers(body []byte, field string) ([]byte, bool) { | ||
| spans := memberSpans(body, strings.Split(field, ".")) | ||
| if len(spans) == 0 { | ||
| // nullMembers returns body with every value that the type error te names | ||
| // replaced by null; t is the type that body decodes into. It reports false | ||
| // when it finds no such value. | ||
| func nullMembers(body []byte, t reflect.Type, te *json.UnmarshalTypeError) ([]byte, bool) { | ||
| w := spanWalk{body: body, dec: json.NewDecoder(bytes.NewReader(body))} | ||
| if w.value(t, strings.Split(te.Field, ".")) != nil || len(w.spans) == 0 { | ||
| return nil, false | ||
| } | ||
| // A map key that holds "." gives two values one path (the keys "a.b" | ||
| // and "a" then "b"): the one that te names has the type it names. When | ||
| // none has it (te names a map key, or comes from an UnmarshalJSON), | ||
| // every value at the path is null. | ||
| if slices.ContainsFunc(w.spans, func(s valueSpan) bool { return s.typ == te.Type }) { | ||
| w.spans = slices.DeleteFunc(w.spans, func(s valueSpan) bool { return s.typ != te.Type }) | ||
| } | ||
| var out bytes.Buffer | ||
| prev := 0 | ||
| for _, sp := range spans { | ||
| out.Write(body[prev:sp[0]]) | ||
| for _, sp := range w.spans { | ||
| out.Write(body[prev:sp.start]) | ||
| out.WriteString("null") | ||
| prev = sp[1] | ||
| prev = sp.end | ||
| } | ||
| out.Write(body[prev:]) | ||
| return out.Bytes(), true | ||
| } | ||
|
|
||
| // memberSpans returns the byte ranges of the values at path in the JSON | ||
| // value b. encoding/json matches a member name to a field without regard to | ||
| // case, and so does this. | ||
| func memberSpans(b []byte, path []string) [][2]int { | ||
| dec := json.NewDecoder(bytes.NewReader(b)) | ||
| open, err := dec.Token() | ||
| if err != nil { | ||
| return nil | ||
| } | ||
| var spans [][2]int | ||
| for index := 0; dec.More(); index++ { | ||
| name := strconv.Itoa(index) | ||
| if open == json.Delim('{') { | ||
| key, err := dec.Token() | ||
| if err != nil { | ||
| return nil | ||
| // valueSpan is the byte range of a value in a body, and the type (without | ||
| // pointers) that the value decodes into. | ||
| type valueSpan struct { | ||
| start, end int | ||
| typ reflect.Type | ||
| } | ||
|
|
||
| // spanWalk finds the values at a path in one pass over a body, following | ||
| // the Go type that the body decodes into. The path is an | ||
| // UnmarshalTypeError.Field split at each ".": encoding/json spells the path | ||
| // of a value as its JSON Pointer (RFC 6901) with each "/" written as ".", so | ||
| // a member is its name with "~" as "~0" and "/" as "~1", and an array | ||
| // element is its index. A name takes one segment more than it holds ".". A | ||
| // member of a struct is the field that encoding/json decodes it into (see | ||
| // fieldFor), and members that differ in case can be one field; a member | ||
| // that is no field is on the path only as a key of the struct's `,embed` | ||
| // map (see bodyFields). A member of a map is its exact key. | ||
| // A type with its own UnmarshalJSON is walked as its fields, which is right | ||
| // for a method that decodes into them (wrefCreateBranchRequest). | ||
| type spanWalk struct { | ||
| body []byte | ||
| dec *json.Decoder | ||
| spans []valueSpan | ||
| } | ||
|
|
||
| // value reads the next value of the body, which decodes into t, and adds | ||
| // the span of each value at path in it. | ||
| func (w *spanWalk) value(t reflect.Type, path []string) error { | ||
| for t.Kind() == reflect.Pointer { | ||
| t = t.Elem() | ||
| } | ||
| // The value starts after the white space, ":" or "," that follow the | ||
| // last token. | ||
| start := len(w.body) - len(bytes.TrimLeft(w.body[w.dec.InputOffset():], " \t\r\n:,")) | ||
| if len(path) == 0 { | ||
| err := w.skip() | ||
| w.spans = append(w.spans, valueSpan{start, int(w.dec.InputOffset()), t}) | ||
| return err | ||
| } | ||
| switch k := t.Kind(); { | ||
| case start == len(w.body): | ||
| case w.body[start] == '{' && (k == reflect.Struct || k == reflect.Map): | ||
| return w.members(t, path) | ||
| case w.body[start] == '[' && (k == reflect.Slice || k == reflect.Array): | ||
| return w.elements(t.Elem(), path) | ||
| } | ||
| return w.skip() | ||
| } | ||
|
|
||
| // members reads an object that decodes into the struct or map t. | ||
| func (w *spanWalk) members(t reflect.Type, path []string) error { | ||
| st := structFields{rest: t} // a map takes every member | ||
| if t.Kind() == reflect.Struct { | ||
| st = bodyFields(t) | ||
| } | ||
| if _, err := w.dec.Token(); err != nil { | ||
| return err | ||
| } | ||
| for w.dec.More() { | ||
| tok, err := w.dec.Token() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| name, _ := tok.(string) | ||
| n := strings.Count(name, ".") + 1 | ||
| var next reflect.Type // nil: the member is not on the path | ||
| if n <= len(path) { | ||
| seg := strings.Join(path[:n], ".") | ||
| if i := fieldFor(st.fields, name); i >= 0 { | ||
| if i == fieldFor(st.fields, strings.ReplaceAll(strings.ReplaceAll(seg, "~1", "/"), "~0", "~")) { | ||
| next = st.fields[i].typ | ||
| } | ||
| } else if st.rest != nil && st.rest.Kind() == reflect.Map && strings.ReplaceAll(strings.ReplaceAll(name, "~", "~0"), "/", "~1") == seg { | ||
| next = st.rest.Elem() | ||
| } | ||
| name, _ = key.(string) | ||
| } | ||
| var raw json.RawMessage | ||
| if dec.Decode(&raw) != nil { | ||
| return nil | ||
| if next == nil { | ||
| err = w.skip() | ||
| } else { | ||
| err = w.value(next, path[n:]) | ||
| } | ||
| end := int(dec.InputOffset()) | ||
| start := end - len(raw) | ||
| if !strings.EqualFold(name, path[0]) { | ||
| continue | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if len(path) == 1 { | ||
| spans = append(spans, [2]int{start, end}) | ||
| continue | ||
| } | ||
| _, err := w.dec.Token() | ||
| return err | ||
| } | ||
|
|
||
| // elements reads an array whose elements decode into t. | ||
| func (w *spanWalk) elements(t reflect.Type, path []string) error { | ||
| if _, err := w.dec.Token(); err != nil { | ||
| return err | ||
| } | ||
| for i := 0; w.dec.More(); i++ { | ||
| var err error | ||
| if strconv.Itoa(i) == path[0] { | ||
| err = w.value(t, path[1:]) | ||
| } else { | ||
| err = w.skip() | ||
| } | ||
| for _, sp := range memberSpans(raw, path[1:]) { | ||
| spans = append(spans, [2]int{start + sp[0], start + sp[1]}) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return spans | ||
| _, err := w.dec.Token() | ||
| return err | ||
| } | ||
|
|
||
| // skip reads the next value of the body. | ||
| func (w *spanWalk) skip() error { | ||
| var raw json.RawMessage | ||
| return w.dec.Decode(&raw) | ||
| } | ||
|
|
||
| // bodyField is a field of a struct type that encoding/json decodes: its | ||
| // JSON name, its type, its depth (more than 0 for a field of an embedded | ||
| // struct), whether its json tag gives the name, and whether the name | ||
| // matches in its own case only (the tag option case:strict). | ||
| type bodyField struct { | ||
| name string | ||
| typ reflect.Type | ||
| depth int | ||
| tagged, strict bool | ||
| } | ||
|
|
||
| // structFields is what bodyFields returns for a struct type. | ||
| type structFields struct { | ||
| fields []bodyField | ||
| rest reflect.Type // the `,embed` map or jsontext.Value that takes the members that no field takes, or nil | ||
| } | ||
|
|
||
| // bodyFields returns the fields of the struct type t that encoding/json | ||
| // decodes, in breadth-first order, by the rules of encoding/json (on | ||
| // json/v2, as Go 1.27 builds it): a struct or a pointer to one that is | ||
| // embedded without a JSON name, or has the option embed and no name, adds | ||
| // its fields one level deeper, and of the fields with one name only the | ||
| // shallowest one counts, or at one depth the only one whose json tag gives | ||
| // the name. Of the fields that take the members that no field takes (see | ||
| // takesRest), the shallowest one does, unless two share its depth. | ||
| func bodyFields(t reflect.Type) structFields { | ||
| if fields, ok := bodyFieldsCache.Load(t); ok { | ||
| return fields.(structFields) | ||
| } | ||
| type embedded struct { | ||
| t reflect.Type | ||
| depth int | ||
| follow bool // false for a type seen before: its embedded structs are not followed | ||
| } | ||
| queue := []embedded{{t, 0, true}} | ||
| seen := map[reflect.Type]bool{t: true} | ||
| var all, rests []bodyField | ||
| for i := 0; i < len(queue); i++ { | ||
| q := queue[i] | ||
| for j := range q.t.NumField() { | ||
| sf := q.t.Field(j) | ||
| tag := sf.Tag.Get("json") | ||
| if tag == "-" || !sf.IsExported() && !sf.Anonymous { | ||
| continue | ||
| } | ||
| name, embed, strict := jsonTag(tag) | ||
| ft := sf.Type | ||
| if ft.Kind() == reflect.Pointer && ft.Name() == "" { | ||
| ft = ft.Elem() | ||
| } | ||
| embed = name == "" && (embed || sf.Anonymous && ft.Kind() == reflect.Struct) | ||
| switch { | ||
| case embed && ft.Kind() == reflect.Struct: | ||
| if q.follow { | ||
| queue = append(queue, embedded{ft, q.depth + 1, !seen[ft]}) | ||
| } | ||
| seen[ft] = true | ||
| case embed && sf.IsExported() && takesRest(ft): | ||
| rests = append(rests, bodyField{typ: ft, depth: q.depth}) | ||
| case sf.IsExported() || ft.Kind() == reflect.Struct: | ||
| all = append(all, bodyField{cmp.Or(name, sf.Name), sf.Type, q.depth, name != "", strict}) | ||
| } | ||
| } | ||
| } | ||
| var fields structFields | ||
| for i, f := range all { | ||
| counts := true | ||
| for j, g := range all { | ||
| if j != i && g.name == f.name && (g.depth < f.depth || g.depth == f.depth && (g.tagged || !f.tagged)) { | ||
| counts = false | ||
| break | ||
| } | ||
| } | ||
| if counts { | ||
| fields.fields = append(fields.fields, f) | ||
| } | ||
| } | ||
| if len(rests) == 1 || len(rests) > 1 && rests[0].depth < rests[1].depth { | ||
| fields.rest = rests[0].typ | ||
| } | ||
| bodyFieldsCache.Store(t, fields) | ||
| return fields | ||
| } | ||
|
|
||
| // bodyFieldsCache holds the bodyFields of each struct type. | ||
| var bodyFieldsCache sync.Map | ||
|
|
||
| // takesRest reports whether encoding/json gives the members that no field | ||
| // takes to an exported field of type t (without pointers) that has the | ||
| // option embed and no name: a jsontext.Value takes them as they are, and so | ||
| // does a map whose key type is a string kind without marshal methods. Any | ||
| // other such field is a plain field. | ||
| func takesRest(t reflect.Type) bool { | ||
| if t.PkgPath() == "encoding/json/jsontext" && t.Name() == "Value" { | ||
| return true | ||
| } | ||
| return t.Kind() == reflect.Map && t.Key().Kind() == reflect.String && !slices.ContainsFunc(marshalMethods, func(m string) bool { | ||
| _, ok := reflect.PointerTo(t.Key()).MethodByName(m) | ||
| return ok | ||
|
Comment on lines
+398
to
+400
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When an embedded map's string-like key type merely declares one of these method names with an unrelated signature, Useful? React with 👍 / 👎. |
||
| }) | ||
| } | ||
|
|
||
| // marshalMethods are the methods by which a type marshals or unmarshals | ||
| // itself in encoding/json. | ||
| var marshalMethods = []string{"MarshalJSON", "MarshalJSONTo", "AppendText", "MarshalText", "UnmarshalJSON", "UnmarshalJSONFrom", "UnmarshalText"} | ||
|
|
||
| // jsonTag returns the JSON name that a json struct tag gives, "" for none, | ||
| // and whether its options hold embed, and case:strict without case:ignore, | ||
| // as encoding/json reads the tag (parseFieldOptions of json/v2). The name | ||
| // ends at the first ",". When a backslash, a quote or a backtick comes | ||
| // first, the name is the run of letters, digits and "_" that starts the | ||
| // tag, and none when that run is empty or starts with a digit. Invalid | ||
| // UTF-8 in the name reads as U+FFFD, as in a member name. | ||
| func jsonTag(tag string) (name string, embed, strict bool) { | ||
| name, opts, _ := strings.Cut(tag, ",") | ||
| if strings.ContainsAny(name, "\\'\"`") { | ||
| name = name[:len(name)-len(strings.TrimLeftFunc(name, func(r rune) bool { | ||
| return r == '_' || unicode.IsLetter(r) || unicode.IsNumber(r) | ||
| }))] | ||
| if r, _ := utf8.DecodeRuneInString(name); unicode.IsNumber(r) { | ||
| name = "" | ||
| } | ||
| } | ||
| ignore := false | ||
| for opt := range strings.SplitSeq(opts, ",") { | ||
| embed = embed || opt == "embed" | ||
| strict = strict || opt == "case:strict" | ||
| ignore = ignore || opt == "case:ignore" | ||
| } | ||
| return string([]rune(name)), embed, strict && !ignore | ||
| } | ||
|
|
||
| // fieldFor returns the index in fields of the field that encoding/json | ||
| // decodes a member name into: the field of that name, else the first one | ||
| // whose name matches it without regard to case and has no case:strict. It | ||
| // returns -1 when no field takes the name. | ||
| func fieldFor(fields []bodyField, name string) int { | ||
| if i := slices.IndexFunc(fields, func(f bodyField) bool { return f.name == name }); i >= 0 { | ||
| return i | ||
| } | ||
| return slices.IndexFunc(fields, func(f bodyField) bool { return !f.strict && strings.EqualFold(f.name, name) }) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When ambiguous paths have the same recursive type, filtering solely by
te.Typekeeps and nulls multiple candidates, including values that have not failed yet. For example, withtype R map[string]R, decoding{"r":{"a":{"b":1},"a.b":{"c":2}},"count":"x"}first reportsr.a.b; both the nested value and the whole"a.b"map have typeR, so this code nulls both and permanently hides the laterr.a.b.cerror. The resulting response therefore omits an invalid field despiteunmarshalBodypromising to name every wrong-type field.Useful? React with 👍 / 👎.