diff --git a/cmd/nntp-fakeserver/config.go b/cmd/nntp-fakeserver/config.go new file mode 100644 index 0000000..91139a7 --- /dev/null +++ b/cmd/nntp-fakeserver/config.go @@ -0,0 +1,474 @@ +package main + +import ( + "bytes" + "crypto/tls" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "math" + "reflect" + "regexp" + "slices" + "sort" + "strings" + "time" + + "github.com/go-while/nntp-testkit/fakeserver" +) + +// config is the JSON document the program reads. Every key is the Go field name of the kit type +// (or of the types below), matched exactly; an unknown key is an error. A field added later to +// Options, StoreOptions, PeerOptions, Faults or StartTLSOptions can be used from JSON at once, +// because the decoder walks those types (decodeValue) and does not mirror them. The types +// below (storeConfig, groupConfig, articleConfig, rawStep) are written by hand: a new field of +// fakeserver.Article or Group needs a line there. +type config struct { + Mode string // "store", "peer", "script" or "raw" + + // Options is fakeserver.Options: Addr, Greeting, NoGreeting, RawStall, TLS (true: implicit + // TLS with the program's certificate), Script (mode script), StoreOpts (mode store), Faults, + // Deadline (ms), MaxConns, MaxConnsGreeting. Store and Handler are not settable. + Options fakeserver.Options + + Store storeConfig // mode store: what the store holds + PeerOptions fakeserver.PeerOptions // mode peer + Raw []rawStep // mode raw: the byte script +} + +// storeConfig is the content of the Store of mode store. +type storeConfig struct { + Groups []groupConfig + Articles []articleConfig +} + +// groupConfig is one group: Created defaults to 2020-01-01 UTC. +type groupConfig struct { + Name, Description string + Created time.Time // RFC 3339 + NoPosting bool // LIST ACTIVE "n" +} + +// articleConfig is one fakeserver.Article filed in Group; a zero Date and Arrived default to +// 2020-01-02T03:04:05Z (so that replies are the same on every run). +type articleConfig struct { + Group string + Number int64 + MessageID string + Headers []string + Body []string + HeadersB64 []string // instead of Headers: each line in base64, for bytes that are not UTF-8 + BodyB64 []string // instead of Body, the same + Date time.Time // RFC 3339 + Arrived time.Time +} + +// rawStep is one step of mode raw: read one client line (unless Expect is null or absent) that +// matches Expect, then the dot block when ReadDot, write the chunks of Send, wait Sleep, then +// act as Then says. +type rawStep struct { + Expect *string // anchored regexp for one client line; null: read nothing + ReadDot bool // after the line, read a dot-terminated block (recorded in articles) + DotMax int // >0 with ReadDot: read only this many lines, then go on + Send []rawChunk // each chunk is its own write + Sleep time.Duration // ms, after sending, before Then + Then string // "", close, rst, hang, starttls or inflate +} + +// rawChunk is bytes to write: B64 (base64) or Text (UTF-8, JSON escapes give CR, LF and so on). +type rawChunk struct { + B64 string + Text string + Gap time.Duration // ms, pause before the chunk + Split int // >0: write in pieces of this many bytes + SplitGap time.Duration // ms, pause between pieces +} + +// thens are the actions after a raw step. +var thens = []string{"", "close", "rst", "hang", "starttls", "inflate"} + +var modes = []string{"store", "peer", "script", "raw"} + +// parseConfig decodes data and checks that the parts fit the mode. tlsCfg is the server config +// that "TLS": true and an empty StartTLS stand for. +func parseConfig(data []byte, tlsCfg *tls.Config) (*config, error) { + var raw json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + var cfg config + d := decoder{tls: tlsCfg} + if err := d.decode(reflect.ValueOf(&cfg).Elem(), raw, "config"); err != nil { + return nil, err + } + if !slices.Contains(modes, cfg.Mode) { + return nil, fmt.Errorf(`config.Mode: want one of %s, got %q`, strings.Join(modes, ", "), cfg.Mode) + } + if err := cfg.check(); err != nil { + return nil, err + } + return &cfg, nil +} + +// check rejects the parts of the config that the mode does not use (a setting that is silently +// ignored is a typo that passes), and bad regexps, base64 and thens in raw mode. +func (cfg *config) check() error { + o := reflect.ValueOf(&cfg.Options).Elem() + parts := []struct { + name string + v reflect.Value + in string // the modes that use it + }{ + {"Options.StoreOpts", o.FieldByName("StoreOpts"), "store"}, + {"Store", reflect.ValueOf(&cfg.Store).Elem(), "store"}, + {"PeerOptions", reflect.ValueOf(&cfg.PeerOptions).Elem(), "peer"}, + {"Options.Script", o.FieldByName("Script"), "script"}, + {"Options.Faults", o.FieldByName("Faults"), "store, peer, script"}, + {"Options.Greeting", o.FieldByName("Greeting"), "store, peer, script"}, + {"Raw", reflect.ValueOf(&cfg.Raw).Elem(), "raw"}, + } + for _, p := range parts { + if !p.v.IsValid() { // a kit field was renamed or removed: this program needs the change + return fmt.Errorf("internal error: config.%s no longer exists in the kit", p.name) + } + if !isEmpty(p.v) && !strings.Contains(p.in, cfg.Mode) { + return fmt.Errorf("config.%s is not used in mode %q (only in %s)", p.name, cfg.Mode, p.in) + } + } + if err := checkFaults(cfg.Options.Faults); err != nil { + return err + } + for i, st := range cfg.Options.Script { + if _, err := regexp.Compile("^(?:" + st.Expect + ")$"); err != nil { + return fmt.Errorf("config.Options.Script[%d].Expect: %w", i, err) + } + } + if cfg.Mode != "raw" { + return nil + } + deadline := cfg.Options.Deadline + if deadline <= 0 { + deadline = defaultDeadline + } + // Conn.Sleep ends every pause at the Deadline, so a longer one would be cut short silently + tooLong := func(field string, d time.Duration) error { + if d > deadline { + return fmt.Errorf("config.%s: %v is longer than Options.Deadline (%v), which ends every pause; raise Options.Deadline", field, d, deadline) + } + return nil + } + for i, st := range cfg.Raw { + if err := tooLong(fmt.Sprintf("Raw[%d].Sleep", i), st.Sleep); err != nil { + return err + } + for j, ch := range st.Send { + if err := errors.Join( + tooLong(fmt.Sprintf("Raw[%d].Send[%d].Gap", i, j), ch.Gap), + tooLong(fmt.Sprintf("Raw[%d].Send[%d].SplitGap", i, j), ch.SplitGap), + ); err != nil { + return err + } + } + if st.Expect != nil { + if _, err := regexp.Compile("^(?:" + *st.Expect + ")$"); err != nil { + return fmt.Errorf("config.Raw[%d].Expect: %w", i, err) + } + } + if !slices.Contains(thens, st.Then) { + return fmt.Errorf("config.Raw[%d].Then: want one of %q, got %q", i, thens, st.Then) + } + for j, ch := range st.Send { + if _, err := ch.bytes(); err != nil { + return fmt.Errorf("config.Raw[%d].Send[%d]: %w", i, j, err) + } + } + } + return nil +} + +// isEmpty reports whether v holds nothing a section would use: zero values, nil pointers, and +// maps and slices without elements (an empty section that a caller always writes is fine). A +// non-nil pointer is not empty, whatever it points to: "StartTLS": {} turns STARTTLS on. +func isEmpty(v reflect.Value) bool { + switch v.Kind() { + case reflect.Map, reflect.Slice: + return v.Len() == 0 + case reflect.Struct: + for i := range v.NumField() { + if !isEmpty(v.Field(i)) { + return false + } + } + return true + } + return v.IsZero() +} + +// checkFaults rejects a command key of Faults that is not upper case: the kit looks commands up +// upper-cased, so {"Delay": {"date": 100}} would never fire. OverrideLine is exempt, the kit +// normalises its keys (whole client lines, "article 2" is fine). +func checkFaults(f fakeserver.Faults) error { + v := reflect.ValueOf(f) + for i := range v.NumField() { + name := v.Type().Field(i).Name + if v.Field(i).Kind() != reflect.Map || name == "OverrideLine" || v.Type().Field(i).Type.Key().Kind() != reflect.String { + continue + } + for _, k := range v.Field(i).MapKeys() { + if key := k.String(); key != strings.ToUpper(key) { + return fmt.Errorf("config.Options.Faults.%s[%q]: command names are upper case (%q)", name, key, strings.ToUpper(key)) + } + } + } + return nil +} + +// bytes returns the data of the chunk. +func (ch rawChunk) bytes() ([]byte, error) { + if ch.B64 != "" && ch.Text != "" { + return nil, fmt.Errorf("B64 and Text are both set") + } + if ch.Text != "" { + return []byte(ch.Text), nil + } + b, err := base64.StdEncoding.DecodeString(ch.B64) + if err != nil { + return nil, fmt.Errorf("B64: %w", err) + } + return b, nil +} + +// buildStore fills a Store from the config. AddArticle panics on a setup error (a number taken, +// a Message-ID that differs from the header): that becomes an error. +func buildStore(sc storeConfig) (st *fakeserver.Store, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("config.Store: %v", r) + } + }() + day := func(h, m, s int, d int) time.Time { return time.Date(2020, 1, d, h, m, s, 0, time.UTC) } + st = fakeserver.NewStore() + for i, g := range sc.Groups { + if g.Name == "" { + return nil, fmt.Errorf("config.Store.Groups[%d].Name is empty", i) + } + created := g.Created + if created.IsZero() { + created = day(0, 0, 0, 1) + } + st.AddGroupAt(g.Name, g.Description, created) + if g.NoPosting { + st.SetPosting(g.Name, false) + } + } + for i, a := range sc.Articles { + if a.Group == "" { + return nil, fmt.Errorf("config.Store.Articles[%d].Group is empty", i) + } + hdr, err := lines("Headers", a.Headers, a.HeadersB64) + if err != nil { + return nil, fmt.Errorf("config.Store.Articles[%d].%w", i, err) + } + body, err := lines("Body", a.Body, a.BodyB64) + if err != nil { + return nil, fmt.Errorf("config.Store.Articles[%d].%w", i, err) + } + art := fakeserver.Article{Number: a.Number, MessageID: a.MessageID, Headers: hdr, Body: body, Date: a.Date, Arrived: a.Arrived} + if art.Date.IsZero() { + art.Date = day(3, 4, 5, 2) + } + if art.Arrived.IsZero() { + art.Arrived = day(3, 4, 5, 2) + } + st.AddArticle(a.Group, art) + } + return st, nil +} + +// lines returns the text lines, or the lines of the base64 form (not both). +func lines(name string, text, b64 []string) ([]string, error) { + if len(b64) == 0 { + return text, nil + } + if len(text) > 0 { + return nil, fmt.Errorf("%s and %sB64 are both set", name, name) + } + out := make([]string, len(b64)) + for i, l := range b64 { + b, err := base64.StdEncoding.DecodeString(l) + if err != nil { + return nil, fmt.Errorf("%sB64[%d]: %w", name, i, err) + } + out[i] = string(b) + } + return out, nil +} + +// decoder fills Go values from JSON by walking their types. +// +// The rules: a struct is a JSON object whose keys are exported field names (exact match; an +// unknown key is an error), a time.Duration is a number of milliseconds (fractions allowed), a +// map has string keys, a slice is an array, scalars are JSON scalars, a type with an +// UnmarshalJSON (time.Time: RFC 3339) uses it, a *tls.Config is true (the program's server +// config) or false. Functions, channels, interfaces and structs without exported fields (Store) +// cannot be set: naming one is an error. JSON null leaves the zero value. +type decoder struct { + tls *tls.Config +} + +var ( + durationType = reflect.TypeFor[time.Duration]() + tlsConfigType = reflect.TypeFor[tls.Config]() + startTLSType = reflect.TypeFor[fakeserver.StartTLSOptions]() + unmarshalerType = reflect.TypeFor[json.Unmarshaler]() +) + +func isNull(raw json.RawMessage) bool { return bytes.Equal(bytes.TrimSpace(raw), []byte("null")) } + +// pathError is an error at a place in the document. +type pathError struct { + path string + err error +} + +func (e *pathError) Error() string { return e.path + ": " + e.err.Error() } +func (e *pathError) Unwrap() error { return e.err } + +// decode fills dst from raw; an error names the innermost path. +func (d *decoder) decode(dst reflect.Value, raw json.RawMessage, path string) error { + err := d.decodeValue(dst, raw, path) + var pe *pathError + if err != nil && !errors.As(err, &pe) { + return &pathError{path, err} + } + return err +} + +func (d *decoder) decodeValue(dst reflect.Value, raw json.RawMessage, path string) error { + t := dst.Type() + switch { + case t == durationType: + var ms float64 + if err := json.Unmarshal(raw, &ms); err != nil { + return fmt.Errorf("want a number of milliseconds") + } + if math.Abs(ms) > float64(math.MaxInt64/int64(time.Millisecond)) { + return fmt.Errorf("%v ms is out of range", ms) + } + dst.SetInt(int64(math.Round(ms * float64(time.Millisecond)))) + return nil + case t.Kind() == reflect.Pointer && t.Elem() == tlsConfigType: + var on bool + if isNull(raw) { + return nil + } + if err := json.Unmarshal(raw, &on); err != nil { + return fmt.Errorf("want true (the server certificate of this program) or false") + } + if on { + dst.Set(reflect.ValueOf(d.tls)) + } + return nil + case isNull(raw): + return nil + case t.Kind() != reflect.Pointer && reflect.PointerTo(t).Implements(unmarshalerType): + return json.Unmarshal(raw, dst.Addr().Interface()) + } + switch t.Kind() { + case reflect.Pointer: + v := reflect.New(t.Elem()) + if err := d.decodeValue(v.Elem(), raw, path); err != nil { + return err + } + dst.Set(v) + case reflect.Struct: + return d.decodeStruct(dst, raw, path) + case reflect.Map: + if t.Key().Kind() != reflect.String { + return fmt.Errorf("a map with %s keys cannot be set from JSON", t.Key()) + } + var m map[string]json.RawMessage + if err := json.Unmarshal(raw, &m); err != nil { + return fmt.Errorf("want an object") + } + out := reflect.MakeMapWithSize(t, len(m)) + for _, k := range sortedKeys(m) { + e := reflect.New(t.Elem()).Elem() + if err := d.decode(e, m[k], fmt.Sprintf("%s[%q]", path, k)); err != nil { + return err + } + out.SetMapIndex(reflect.ValueOf(k).Convert(t.Key()), e) + } + dst.Set(out) + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { // []byte is a base64 string, as in encoding/json + if !bytes.HasPrefix(bytes.TrimSpace(raw), []byte(`"`)) || json.Unmarshal(raw, dst.Addr().Interface()) != nil { + return fmt.Errorf("want a base64 string") + } + return nil + } + var arr []json.RawMessage + if err := json.Unmarshal(raw, &arr); err != nil { + return fmt.Errorf("want an array") + } + out := reflect.MakeSlice(t, len(arr), len(arr)) + for i, r := range arr { + if err := d.decode(out.Index(i), r, fmt.Sprintf("%s[%d]", path, i)); err != nil { + return err + } + } + dst.Set(out) + case reflect.Bool, reflect.String, reflect.Float32, reflect.Float64, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if err := json.Unmarshal(raw, dst.Addr().Interface()); err != nil { + return fmt.Errorf("want %s", t.Kind()) + } + default: + return fmt.Errorf("cannot be set from JSON (type %s)", t) + } + return nil +} + +func (d *decoder) decodeStruct(dst reflect.Value, raw json.RawMessage, path string) error { + t := dst.Type() + var known []string + for i := range t.NumField() { + if t.Field(i).IsExported() { + known = append(known, t.Field(i).Name) + } + } + if len(known) == 0 { + return fmt.Errorf("cannot be set from JSON (type %s)", t) + } + var m map[string]json.RawMessage + if err := json.Unmarshal(raw, &m); err != nil { + return fmt.Errorf("want an object") + } + if t == startTLSType { + // STARTTLS needs a config: this program's, unless "Config": false switches it off. + if _, set := m["Config"]; !set { + dst.FieldByName("Config").Set(reflect.ValueOf(d.tls)) + } + } + for _, k := range sortedKeys(m) { + f, ok := t.FieldByName(k) + if !ok || !f.IsExported() || len(f.Index) != 1 { + return &pathError{path + "." + k, fmt.Errorf("unknown key (the keys of %s are %s)", t.Name(), strings.Join(known, ", "))} + } + if err := d.decode(dst.FieldByIndex(f.Index), m[k], path+"."+k); err != nil { + return err + } + } + return nil +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/cmd/nntp-fakeserver/main.go b/cmd/nntp-fakeserver/main.go new file mode 100644 index 0000000..091fe00 --- /dev/null +++ b/cmd/nntp-fakeserver/main.go @@ -0,0 +1,324 @@ +// Command nntp-fakeserver runs the kit's fake NNTP server (package fakeserver) as a process that +// a JSON file configures, so test suites in other languages (Python, shell, C) can use it. +// +// nntp-fakeserver [-listen host:port] config.json +// +// Protocol with the caller: once the server listens, one line of JSON goes to stdout, +// +// {"addr":"127.0.0.1:41233","host":"127.0.0.1","port":41233,"ca":"/tmp/nntp-fakeserver-1/ca.pem","ca_pem":"-----BEGIN CERTIFICATE-----\n..."} +// +// ca is a PEM file that holds the certificate of the server (it is self-signed, valid for +// localhost, 127.0.0.1 and ::1), ca_pem its text: a client verifies implicit TLS and STARTTLS +// against it. The +// server then runs until stdin reaches end of file (or the process gets SIGINT or SIGTERM). It +// prints one more line of JSON, the summary, removes its temporary directory and exits 0: +// +// {"lines":[...],"mismatches":[...],"articles":[[...]],"notes":[...],"accepted":1, +// "counts":{"GROUP":1},"tls_handshakes":0,"tls_failed":0,"server_names":[], +// "lines_b64":[...],"mismatches_b64":[...],"articles_raw":[...],"max_open":1,"closed":1} +// +// lines are the client lines of all connections, mismatches those that did not fit (raw mode, +// script mode), articles the dot blocks received (POST, IHAVE, TAKETHIS, raw ReadDot), notes +// what the server itself ran into (a failed handshake, a cut-off block), counts the number of +// lines per upper-cased command, tls_handshakes and tls_failed the TLS handshakes (implicit and +// STARTTLS) and server_names the SNI of every ClientHello ("" when none). Exit status 2 and a +// message on stderr for a bad config or flag, 1 for another failure. +// +// A JSON string is UTF-8, so lines, mismatches and articles show any other byte (8-bit text, +// yEnc) as U+FFFD. lines_b64 and mismatches_b64 hold the same lines, articles_raw the blocks +// (the exact bytes up to the "." line, still dot-stuffed, one entry per block), all in base64 +// and exact. max_open is the most connections open at once, closed the number that ended. +// +// The config is one JSON object. Its keys are the Go field names of the kit types, matched +// exactly, and a key that does not exist is an error (a typo fails at once). Durations are +// numbers of milliseconds (fractions allowed). Mode says what runs: +// +// store Store + Options.StoreOpts + Options.Faults: the RFC 3977 reader of the kit +// peer PeerOptions: a transit peer (IHAVE, CHECK, TAKETHIS), Options.Faults apply to it +// script Options.Script: lines of text, as fakeserver.Step +// raw Raw: a byte-exact script, run by this program +// +// Common to all modes is Options (fakeserver.Options): Addr, Greeting, NoGreeting, RawStall, +// TLS (true: implicit TLS from the first byte, with the certificate behind ca), Deadline (ms, +// default 10000: bounds every server-side wait and each TLS handshake), MaxConns and +// MaxConnsGreeting. A StartTLS that is {} (or has only the knobs Refuse, HideCapability, +// AuthNeedsTLS, CloseAfterHandshake) offers STARTTLS with the same certificate. Functions, +// channels and the Store itself cannot be set from JSON. A part that the mode does not use is +// an error, unless it is empty ({} or []). "StartTLS": {"Config": false} switches STARTTLS off +// again. The command names that key the maps of Faults are upper case (a lower case key is an +// error: it would never fire), except OverrideLine, which takes whole client lines. +// +// Store mode, with a fault and STARTTLS: +// +// {"Mode": "store", +// "Options": { +// "Greeting": "200 hello", +// "Faults": {"Delay": {"BODY": 2500}, "DropInMultiline": {"ARTICLE": 2}, "LongLine": {"BODY": 65535}}, +// "StoreOpts": {"User": "u", "Pass": "p", "Compress": true, "StartTLS": {"HideCapability": true}}}, +// "Store": { +// "Groups": [{"Name": "alt.test", "Description": "test group"}], +// "Articles": [{"Group": "alt.test", "Number": 1, "MessageID": "<1@test>", +// "Headers": ["Subject: one", "From: a@b.example"], "Body": ["text", ".dot"]}]}} +// +// Groups take Name, Description, Created (RFC 3339, default 2020-01-01) and NoPosting; +// Articles take Group, Number, MessageID, Headers, Body (lines, no CRLF, not dot-stuffed), Date +// and Arrived (RFC 3339, default 2020-01-02T03:04:05Z). HeadersB64 and BodyB64 are Headers and +// Body with every line in base64, for bytes that are not UTF-8 (use one form of each). The +// replies of Faults (Override and so on) are UTF-8 only; raw mode with B64 chunks writes any +// bytes. +// +// Peer mode: +// +// {"Mode": "peer", "PeerOptions": {"Have": {"": true}, "Reject": {"": true}, +// "Defer436Once": {"": true}, "User": "u", "Pass": "p", "Compress": true}} +// +// Script mode: +// +// {"Mode": "script", "Options": {"Script": [{"Expect": "CAPABILITIES", "Reply": ["101 caps", "VERSION 2", "."]}]}} +// +// Raw mode: Raw is a list of steps. Each step reads one client line that matches Expect (an +// anchored regexp; null or absent: read nothing), then a dot block when ReadDot (DotMax > 0: +// only that many lines), writes its Send chunks, waits Sleep ms and then does Then: "" (go on), +// "close", "rst" (reset the TCP connection), "hang" (keep reading, never answer), "starttls" +// (handshake on the open connection, then read through TLS) or "inflate" (read the client +// through raw DEFLATE from now on; the compressed replies are in the script). A chunk is Text +// (UTF-8) or B64 (base64), written after Gap ms, in one write or in pieces of Split bytes with +// SplitGap ms between them. A line that does not match gets "500 unexpected ..." and is a +// mismatch, as is every line after the last step. Pauses, reads of an expected line and +// writes are bounded by Options.Deadline and end when the server closes (hang and the lines +// after the script wait for the client without a limit): a client that stays silent longer than +// the Deadline before an expected line is cut off, and a Sleep, Gap or SplitGap longer than the +// Deadline is a config error. A line over 1 MiB, a dot block over 16 MiB or more than 64 MiB on +// one connection close the connection and add a note. The script writes the greeting itself (a +// first step with Expect null). +// +// {"Mode": "raw", "Raw": [ +// {"Send": [{"Text": "200 raw ready\r\n"}]}, +// {"Expect": "GROUP .*", "Send": [{"Text": "211 1 1 1 alt.a\r\n", "Split": 3, "SplitGap": 1}]}, +// {"Expect": "STARTTLS", "Send": [{"Text": "382 go\r\n"}], "Then": "starttls"}, +// {"Expect": "QUIT", "Send": [{"B64": "MjA1IGJ5ZQ0K"}], "Then": "close"}]} +// +// The certificate (tlscert.GenerateSelfSigned) and the CA file are made on every start and live +// in a temporary directory that is removed on exit. +package main + +import ( + "context" + "crypto/tls" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "syscall" + "time" + + "github.com/go-while/nntp-testkit/fakeserver" + "github.com/go-while/nntp-testkit/tlscert" +) + +// defaultDeadline is Options.Deadline when the config sets none. +const defaultDeadline = 10 * time.Second + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + go func() { // after the first signal the next one kills the process, as it should while shutting down + <-ctx.Done() + stop() + }() + os.Exit(run(ctx, os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) +} + +// ready is the line on stdout when the server listens. +type ready struct { + Addr string `json:"addr"` + Host string `json:"host"` + Port int `json:"port"` + CA string `json:"ca"` + // CAPEM is the text of the CA file, for a caller that wants the certificate in memory. + CAPEM string `json:"ca_pem"` +} + +// summary is the line on stdout when the server has stopped. +type summary struct { + Lines []string `json:"lines"` + Mismatches []string `json:"mismatches"` + Articles [][]string `json:"articles"` + Notes []string `json:"notes"` + Accepted int `json:"accepted"` + Counts map[string]int `json:"counts"` + TLSHandshakes int `json:"tls_handshakes"` + TLSFailed int `json:"tls_failed"` + ServerNames []string `json:"server_names"` + + // The lossless forms: JSON strings are UTF-8, so lines, mismatches and articles hold U+FFFD + // for any other byte. These hold the bytes as sent, in base64. + LinesB64 []string `json:"lines_b64"` // lines, one base64 string per line + MismatchesB64 []string `json:"mismatches_b64"` // mismatches + ArticlesRaw []string `json:"articles_raw"` // per block: the exact bytes up to the "." line, still dot-stuffed + + MaxOpen int `json:"max_open"` // the most connections open at once + Closed int `json:"closed"` // connections that have ended +} + +// run is main without os.Exit: it serves until stdin reaches end of file or ctx is done, and +// returns the exit status. +func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("nntp-fakeserver", flag.ContinueOnError) + fs.SetOutput(stderr) + listen := fs.String("listen", "", "listen address `host:port` (default Options.Addr of the config, else 127.0.0.1:0)") + fs.Usage = func() { + fmt.Fprintf(stderr, "usage: nntp-fakeserver [flags] config.json\n\nflags:\n") + fs.PrintDefaults() + fmt.Fprintf(stderr, "\nprints {addr, host, port, ca, ca_pem} when listening,\n"+ + "serves until stdin reaches end of file,\n"+ + "then prints {lines, lines_b64, mismatches, mismatches_b64, articles, articles_raw, notes,\n"+ + "accepted, counts, tls_handshakes, tls_failed, server_names, max_open, closed} and exits;\n"+ + "see the package documentation for the config\n") + } + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + fail := func(code int, f string, a ...any) int { + fmt.Fprintf(stderr, "nntp-fakeserver: "+f+"\n", a...) + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(stderr, "nntp-fakeserver: want exactly one argument, the config file") + fs.Usage() + return 2 + } + data, err := os.ReadFile(fs.Arg(0)) + if err != nil { + return fail(2, "%v", err) + } + + dir, err := os.MkdirTemp("", "nntp-fakeserver-") + if err != nil { + return fail(1, "%v", err) + } + defer os.RemoveAll(dir) + leaf, err := tlscert.GenerateSelfSigned(tlscert.Options{CommonName: "nntp-fakeserver"}) + if err != nil { + return fail(1, "%v", err) + } + files, err := leaf.WriteFiles(dir, 0o600) + if err != nil { + return fail(1, "%v", err) + } + tlsCfg := leaf.ServerConfig() + + cfg, err := parseConfig(data, tlsCfg) + if err != nil { + return fail(2, "%s: %v", fs.Arg(0), err) + } + var store *fakeserver.Store + if cfg.Mode == "store" { + if store, err = buildStore(cfg.Store); err != nil { + return fail(2, "%s: %v", fs.Arg(0), err) + } + } + srv, rec, err := start(cfg, store, *listen, tlsCfg) + if err != nil { + return fail(1, "%v", err) + } + defer srv.Close() + + out := json.NewEncoder(stdout) + out.SetEscapeHTML(false) + if err := out.Encode(ready{Addr: srv.Addr(), Host: srv.Host(), Port: srv.Port(), CA: files.CA, CAPEM: string(leaf.CAPEM)}); err != nil { + return fail(1, "stdout: %v", err) + } + + eof := make(chan struct{}) + go func() { // the caller closes stdin when its client is done + _, _ = io.Copy(io.Discard, stdin) + close(eof) + }() + select { + case <-eof: + case <-ctx.Done(): + } + srv.Close() // waits for the connection goroutines: what they recorded is complete + + rec.mu.Lock() + defer rec.mu.Unlock() + s := summary{ + Lines: srv.Lines(), Mismatches: srv.Mismatches(), Articles: srv.Articles(), Notes: rec.notes, + Accepted: srv.Accepted(), Counts: srv.Counts(), + TLSHandshakes: srv.TLSHandshakes(), TLSFailed: srv.TLSFailed(), ServerNames: srv.TLSServerNames(), + MaxOpen: srv.MaxOpen(), Closed: srv.Closed(), + } + raws := srv.ArticlesRaw() + if cfg.Mode == "raw" { + s.Lines, s.Mismatches, s.Articles, s.Counts = rec.lines, rec.mismatches, rec.articles, rec.counts + raws = rec.articleRaw + } + s.LinesB64, s.MismatchesB64, s.ArticlesRaw = b64s(s.Lines), b64s(s.Mismatches), []string{} + for _, b := range raws { + s.ArticlesRaw = append(s.ArticlesRaw, base64.StdEncoding.EncodeToString(b)) + } + s.Lines, s.Mismatches, s.Notes, s.ServerNames = nz(s.Lines), nz(s.Mismatches), nz(s.Notes), nz(s.ServerNames) + if s.Articles == nil { + s.Articles = [][]string{} + } + if s.Counts == nil { + s.Counts = map[string]int{} + } + if err := out.Encode(s); err != nil { + return fail(1, "stdout: %v", err) + } + return 0 +} + +// start starts the server of cfg (store: the Store of mode store); listen, when not empty, +// overrides Options.Addr. +func start(cfg *config, store *fakeserver.Store, listen string, tlsCfg *tls.Config) (*fakeserver.Server, *recorder, error) { + o := cfg.Options + if listen != "" { + o.Addr = listen + } + if o.Deadline <= 0 { + o.Deadline = defaultDeadline + } + rec := &recorder{} + var srv *fakeserver.Server + var err error + switch cfg.Mode { + case "store": + o.Store = store + srv, err = fakeserver.Start(o) + case "peer": + po := cfg.PeerOptions + srv, err = fakeserver.StartPeer(o, &po) + case "script": + srv, err = fakeserver.Start(o) + case "raw": + o.NoGreeting = true // the script writes the greeting + o.Handler = rawHandler(cfg.Raw, rec, tlsCfg, o.Deadline) + srv, err = fakeserver.Start(o) + } + return srv, rec, err +} + +// b64s returns the strings in base64 (never nil). +func b64s(s []string) []string { + out := make([]string, len(s)) + for i, l := range s { + out[i] = base64.StdEncoding.EncodeToString([]byte(l)) + } + return out +} + +func nz(s []string) []string { + if s == nil { + return []string{} + } + return s +} diff --git a/cmd/nntp-fakeserver/main_test.go b/cmd/nntp-fakeserver/main_test.go new file mode 100644 index 0000000..490869b --- /dev/null +++ b/cmd/nntp-fakeserver/main_test.go @@ -0,0 +1,1051 @@ +package main + +import ( + "bufio" + "bytes" + "compress/flate" + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "reflect" + "slices" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/go-while/nntp-testkit/client" + "github.com/go-while/nntp-testkit/fakeserver" +) + +// proc is one in-process run of the program, driven as a caller in another language would: the +// config in a file, the ready line on stdout, stdin closed when the client is done. +type proc struct { + t *testing.T + ready ready + stdin *io.PipeWriter + out *bufio.Reader + stderr *bytes.Buffer + code chan int + cancel context.CancelFunc +} + +func writeConfig(t *testing.T, cfg string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(p, []byte(cfg), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +// startProc starts run with the config and waits for the ready line. +func startProc(t *testing.T, cfg string, flags ...string) *proc { + t.Helper() + p := launch(t, cfg, flags...) + line, err := p.out.ReadString('\n') + if err != nil { + t.Fatalf("no ready line: %v\nstderr: %s", err, p.stderr.String()) + } + if err := json.Unmarshal([]byte(line), &p.ready); err != nil { + t.Fatalf("ready line %q: %v", line, err) + } + return p +} + +// launch starts run in a goroutine; the caller reads p.out. +func launch(t *testing.T, cfg string, flags ...string) *proc { + t.Helper() + inR, inW := io.Pipe() + outR, outW := io.Pipe() + ctx, cancel := context.WithCancel(context.Background()) + p := &proc{t: t, stdin: inW, out: bufio.NewReader(outR), stderr: &bytes.Buffer{}, code: make(chan int, 1), cancel: cancel} + args := append(slices.Clone(flags), writeConfig(t, cfg)) + go func() { + code := run(ctx, args, inR, outW, p.stderr) + _ = outW.Close() + p.code <- code + }() + t.Cleanup(func() { + cancel() + _ = inW.Close() + _ = outR.Close() + }) + return p +} + +// finish closes stdin like the caller does and returns the summary and the exit status. +func (p *proc) finish() (summary, int) { + p.t.Helper() + _ = p.stdin.Close() + line, err := p.out.ReadString('\n') + if err != nil { + p.t.Fatalf("no summary line: %v\nstderr: %s", err, p.stderr.String()) + } + var s summary + dec := json.NewDecoder(strings.NewReader(line)) + dec.DisallowUnknownFields() + if err := dec.Decode(&s); err != nil { + p.t.Fatalf("summary %q: %v", line, err) + } + select { + case code := <-p.code: + return s, code + case <-time.After(10 * time.Second): + p.t.Fatal("run did not return") + return s, -1 + } +} + +func (p *proc) dial() *client.Conn { + p.t.Helper() + c, err := client.TryDial(p.ready.Addr, 10*time.Second) + if err != nil { + p.t.Fatalf("dial: %v", err) + } + p.t.Cleanup(c.Close) + return c +} + +func (p *proc) caPool() *x509.CertPool { + p.t.Helper() + pem, err := os.ReadFile(p.ready.CA) + if err != nil { + p.t.Fatal(err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + p.t.Fatalf("no certificate in %s", p.ready.CA) + } + return pool +} + +func cmd(t *testing.T, c *client.Conn, want int, format string, args ...any) string { + t.Helper() + code, line, err := c.TryCmd(format, args...) + if err != nil || code != want { + t.Fatalf("%s: got %d %q err %v, want %d", format, code, line, err, want) + } + return line +} + +const storeCfg = `{"Mode": "store", + "Options": {"Faults": {"DropInMultiline": {"ARTICLE": 2}, "Delay": {"DATE": 300}}}, + "Store": { + "Groups": [{"Name": "alt.test", "Description": "test group"}, {"Name": "alt.empty"}], + "Articles": [ + {"Group": "alt.test", "Number": 1, "MessageID": "<1@test>", + "Headers": ["Subject: one", "From: a@b.example"], "Body": ["text", ".dot"]}, + {"Group": "alt.test", "Number": 2, "MessageID": "<2@test>", + "Headers": ["Subject: two"], "Body": ["b"], "Date": "2021-03-04T05:06:07Z"}]}}` + +// TestStoreWithFaults: the Store is served, a fault applies (a drop in the second line of a +// reply, a delay in ms), and the summary says what the server saw. +func TestStoreWithFaults(t *testing.T) { + p := startProc(t, storeCfg) + if p.ready.Host != "127.0.0.1" || p.ready.Addr != net.JoinHostPort(p.ready.Host, strconv.Itoa(p.ready.Port)) || p.ready.CA == "" { + t.Errorf("ready line %+v", p.ready) + } + c := p.dial() + if line := cmd(t, c, 211, "GROUP alt.test"); line != "211 2 1 2 alt.test" { + t.Errorf("GROUP: %q", line) + } + start := time.Now() + cmd(t, c, 111, "DATE") + if d := time.Since(start); d < 250*time.Millisecond { + t.Errorf("Delay DATE 300 ms took %v", d) + } + cmd(t, c, 220, "ARTICLE 1") + lines, err := c.TryReadDotLines() + if err == nil { + t.Fatalf("ARTICLE was not cut: %q", lines) + } + s, code := p.finish() + if code != 0 { + t.Fatalf("exit %d, stderr %s", code, p.stderr.String()) + } + if want := []string{"GROUP alt.test", "DATE", "ARTICLE 1"}; !slices.Equal(s.Lines, want) { + t.Errorf("lines %q, want %q", s.Lines, want) + } + if s.Accepted != 1 || s.Counts["GROUP"] != 1 || s.Counts["ARTICLE"] != 1 { + t.Errorf("accepted %d counts %v", s.Accepted, s.Counts) + } + if len(s.Mismatches) != 0 || len(s.Notes) != 0 || len(s.Articles) != 0 || s.TLSHandshakes != 0 { + t.Errorf("summary %+v", s) + } + if _, err := os.Stat(p.ready.CA); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the CA file is still there after exit: %v", err) + } +} + +// TestStoreDates: Date, Arrived and Created are RFC 3339 strings; the defaults are fixed. +func TestStoreDates(t *testing.T) { + p := startProc(t, storeCfg) + c := p.dial() + cmd(t, c, 211, "GROUP alt.test") + cmd(t, c, 224, "OVER 1-2") + lines, err := c.TryReadDotLines() + if err != nil || len(lines) != 2 { + t.Fatalf("OVER: %q %v", lines, err) + } + if !strings.Contains(lines[1], "04 Mar 2021 05:06:07") { + t.Errorf("article 2 has its own Date: %q", lines[1]) + } + if !strings.Contains(lines[0], "02 Jan 2020 03:04:05") { + t.Errorf("article 1 has the default Date: %q", lines[0]) + } + p.finish() +} + +// TestPeerIhave: IHAVE against the peer, with its own PeerOptions, and the article is in the +// summary. +func TestPeerIhave(t *testing.T) { + p := startProc(t, `{"Mode": "peer", "PeerOptions": {"Have": {"": true}, "Defer436Once": {"": true}}}`) + c := p.dial() + cmd(t, c, 335, "IHAVE ") + if err := c.Send("Subject: s\r\n\r\n..dot\r\n.\r\n"); err != nil { + t.Fatal(err) + } + code, line, err := c.ReadStatus() + if err != nil || code != 235 { + t.Fatalf("after the article: %d %q %v", code, line, err) + } + cmd(t, c, 435, "IHAVE ") + cmd(t, c, 436, "IHAVE ") + cmd(t, c, 335, "IHAVE ") + s, _ := p.finish() + if len(s.Articles) != 1 || !slices.Equal(s.Articles[0], []string{"Subject: s", "", "..dot"}) { + t.Errorf("articles %q", s.Articles) + } + if s.Counts["IHAVE"] != 4 { + t.Errorf("counts %v", s.Counts) + } +} + +// TestPeerBadOption: a PeerOptions key that is a function or does not exist is refused. +func TestPeerBadOption(t *testing.T) { + for key, want := range map[string]string{ + `"OnArticle": {}`: "cannot be set from JSON", + `"HoldGreeting": true`: "cannot be set from JSON", + `"Have": 3`: "want an object", + } { + p := launch(t, `{"Mode": "peer", "PeerOptions": {`+key+`}}`) + if code := <-p.code; code != 2 || !strings.Contains(p.stderr.String(), want) { + t.Errorf("%s: exit %d, stderr %s", key, code, p.stderr.String()) + } + } +} + +// TestScriptMode: the kit's text script. +func TestScriptMode(t *testing.T) { + p := startProc(t, `{"Mode": "script", "Options": {"Greeting": "200 hi", + "Script": [{"Expect": "CAPABILITIES", "Delay": 20, "Reply": ["101 caps", "VERSION 2", "."]}, + {"Expect": "DATE", "Reply": ["111 20200102030405"]}]}}`) + c := p.dial() + if c.Greeting != "200 hi" { + t.Errorf("greeting %q", c.Greeting) + } + cmd(t, c, 101, "CAPABILITIES") + if _, err := c.TryReadDotLines(); err != nil { + t.Fatal(err) + } + cmd(t, c, 500, "NOPE") + cmd(t, c, 111, "DATE") + s, _ := p.finish() + if !slices.Equal(s.Mismatches, []string{"NOPE"}) { + t.Errorf("mismatches %q", s.Mismatches) + } +} + +const rawCfg = `{"Mode": "raw", "Raw": [ + {"Send": [{"Text": "200 raw ready\r\n"}]}, + {"Expect": "GROUP .*", "Send": [{"Text": "211 1 1 1 alt.a\r\n", "Split": 4, "SplitGap": 2, "Gap": 5}]}, + {"Expect": "POST", "ReadDot": false, "Send": [{"Text": "340 go\r\n"}]}, + {"Expect": "(?i)subject: .*", "ReadDot": true, "DotMax": 3, "Send": [{"B64": "MjQwIG9rDQo="}]}, + {"Expect": "QUIT", "Send": [{"Text": "205 bye\r\n"}], "Then": "close"}]}` + +// TestRawChunks: a reply in pieces arrives whole, base64 chunks are bytes, a dot block is read +// and recorded, a line that does not fit is a mismatch. +func TestRawChunks(t *testing.T) { + p := startProc(t, rawCfg) + c := p.dial() + cmd(t, c, 500, "NOPE") + cmd(t, c, 211, "GROUP alt.a") + cmd(t, c, 340, "POST") + if err := c.Send("Subject: x\r\n\r\nbody\r\n.\r\n"); err != nil { + t.Fatal(err) + } + if code, line, err := c.ReadStatus(); err != nil || code != 240 { + t.Fatalf("%d %q %v", code, line, err) + } + cmd(t, c, 205, "QUIT") + if _, err := c.ReadLine(); !errors.Is(err, io.EOF) { + t.Errorf("after the close: %v", err) + } + s, code := p.finish() + if code != 0 { + t.Fatalf("exit %d", code) + } + if want := []string{"NOPE", "GROUP alt.a", "POST", "Subject: x", "QUIT"}; !slices.Equal(s.Lines, want) { + t.Errorf("lines %q, want %q", s.Lines, want) + } + if !slices.Equal(s.Mismatches, []string{"NOPE"}) { + t.Errorf("mismatches %q", s.Mismatches) + } + if len(s.Articles) != 1 || !slices.Equal(s.Articles[0], []string{"", "body", "."}) { + t.Errorf("articles %q", s.Articles) + } + if s.Counts["GROUP"] != 1 || s.Counts["QUIT"] != 1 { + t.Errorf("counts %v", s.Counts) + } +} + +// TestRawSplitPieces: Split writes one piece per write, so the client sees several reads when +// it reads between them (a pause of 50 ms is far more than the server needs between writes). +func TestRawSplitPieces(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 abcdef\r\n", "Split": 4, "SplitGap": 50}]}]}`) + nc, err := net.DialTimeout("tcp", p.ready.Addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + var reads int + var got []byte + buf := make([]byte, 64) + _ = nc.SetDeadline(time.Now().Add(5 * time.Second)) + for !bytes.HasSuffix(got, []byte("\r\n")) { + n, err := nc.Read(buf) + if err != nil { + t.Fatalf("after %q: %v", got, err) + } + reads++ + got = append(got, buf[:n]...) + } + if string(got) != "200 abcdef\r\n" || reads < 2 { + t.Errorf("got %q in %d reads", got, reads) + } + p.finish() +} + +const rawTLSCfg = `{"Mode": "raw", "Raw": [ + {"Send": [{"Text": "200 raw ready\r\n"}]}, + {"Expect": "STARTTLS", "Send": [{"Text": "382 go\r\n"}], "Then": "starttls"}, + {"Expect": "DATE", "Send": [{"Text": "111 20200102030405\r\n"}]}]}` + +// TestRawStartTLS: a raw script upgrades with the program's certificate; the client verifies it +// against the CA file of the ready line, and the summary counts the handshake and the name. +func TestRawStartTLS(t *testing.T) { + p := startProc(t, rawTLSCfg) + c := p.dial() + tc, err := c.TryStartTLS(&tls.Config{RootCAs: p.caPool(), ServerName: "localhost"}) + if err != nil { + t.Fatal(err) + } + defer tc.Close() + if line := cmd(t, tc, 111, "DATE"); line != "111 20200102030405" { + t.Errorf("DATE over TLS: %q", line) + } + s, _ := p.finish() + if want := []string{"STARTTLS", "DATE"}; !slices.Equal(s.Lines, want) { + t.Errorf("lines %q, want %q", s.Lines, want) + } + if s.TLSHandshakes != 1 || !slices.Equal(s.ServerNames, []string{"localhost"}) || len(s.Mismatches) != 0 { + t.Errorf("tls_handshakes %d server_names %q mismatches %q notes %q", s.TLSHandshakes, s.ServerNames, s.Mismatches, s.Notes) + } +} + +// TestRawStartTLSPipelined: plaintext sent before the handshake is noted and the connection is +// closed, as the kit does in Store mode. +func TestRawStartTLSPipelined(t *testing.T) { + p := startProc(t, rawTLSCfg) + nc, err := net.DialTimeout("tcp", p.ready.Addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + _ = nc.SetDeadline(time.Now().Add(5 * time.Second)) + br := bufio.NewReader(nc) + if _, err := br.ReadString('\n'); err != nil { + t.Fatal(err) + } + // Both lines go in one write, so the second is already in the server's read buffer when + // the step for STARTTLS has answered. + if _, err := io.WriteString(nc, "STARTTLS\r\nDATE\r\n"); err != nil { + t.Fatal(err) + } + if _, err := io.Copy(io.Discard, br); err != nil && !errors.Is(err, syscall.ECONNRESET) { + t.Fatal(err) + } + s, _ := p.finish() + if len(s.Notes) != 1 || !strings.Contains(s.Notes[0], "pipelined") { + t.Errorf("notes %q", s.Notes) + } +} + +// TestRawRST: Then "rst" resets the connection. +func TestRawRST(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, + {"Expect": "QUIT", "Sleep": 20, "Then": "rst"}]}`) + c := p.dial() + if err := c.Send("QUIT\r\n"); err != nil { + t.Fatal(err) + } + _, err := c.ReadLine() + if !errors.Is(err, syscall.ECONNRESET) { + t.Errorf("read after rst: %v, want connection reset", err) + } + p.finish() +} + +// TestRawInflate: inflate reads the client through raw DEFLATE from then on (the replies stay +// plain here). +func TestRawInflate(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, + {"Expect": "COMPRESS DEFLATE", "Send": [{"Text": "206 ok\r\n"}], "Then": "inflate"}, + {"Expect": "DATE", "Send": [{"Text": "111 20200102030405\r\n"}]}]}`) + c := p.dial() + cmd(t, c, 206, "COMPRESS DEFLATE") + var buf bytes.Buffer + fw, err := flate.NewWriter(&buf, flate.DefaultCompression) + if err != nil { + t.Fatal(err) + } + fw.Write([]byte("DATE\r\n")) + if err := fw.Flush(); err != nil { + t.Fatal(err) + } + if err := c.Send(buf.String()); err != nil { + t.Fatal(err) + } + if code, line, err := c.ReadStatus(); err != nil || code != 111 { + t.Fatalf("DATE through DEFLATE: %d %q %v", code, line, err) + } + s, _ := p.finish() + if want := []string{"COMPRESS DEFLATE", "DATE"}; !slices.Equal(s.Lines, want) || len(s.Mismatches) != 0 { + t.Errorf("lines %q mismatches %q", s.Lines, s.Mismatches) + } +} + +// TestRawHang: hang reads and never answers, and does not close. +func TestRawHang(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, {"Expect": "QUIT", "Then": "hang"}]}`) + nc, err := net.DialTimeout("tcp", p.ready.Addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + br := bufio.NewReader(nc) + if _, err := br.ReadString('\n'); err != nil { + t.Fatal(err) + } + io.WriteString(nc, "QUIT\r\nMORE\r\n") + _ = nc.SetReadDeadline(time.Now().Add(300 * time.Millisecond)) + var ne net.Error + if line, err := br.ReadString('\n'); !errors.As(err, &ne) || !ne.Timeout() { + t.Errorf("a hanging server answered or closed: %q %v", line, err) + } + p.finish() +} + +// TestImplicitTLS: Options.TLS makes every connection TLS; the CA file verifies for localhost +// and for 127.0.0.1, and the handshakes and names are in the summary. +func TestImplicitTLS(t *testing.T) { + p := startProc(t, `{"Mode": "store", "Options": {"TLS": true}, "Store": {"Groups": [{"Name": "a.b"}]}}`) + pool := p.caPool() + for _, name := range []string{"localhost", "127.0.0.1"} { + nc, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", p.ready.Addr, &tls.Config{RootCAs: pool, ServerName: name}) + if err != nil { + t.Fatalf("verify %s: %v", name, err) + } + c := client.New(t, nc) + if _, err := c.ReadGreeting(); err != nil { + t.Fatal(err) + } + cmd(t, c, 211, "GROUP a.b") + c.Close() + } + // A client that does not trust the certificate fails; the server counts the failure. + if nc, err := tls.Dial("tcp", p.ready.Addr, &tls.Config{ServerName: "localhost"}); err == nil { + nc.Close() + t.Error("the system roots accepted the certificate") + } + s, _ := p.finish() + if s.TLSHandshakes != 2 || s.TLSFailed != 1 { + t.Errorf("tls_handshakes %d tls_failed %d", s.TLSHandshakes, s.TLSFailed) + } + if len(s.ServerNames) != 3 || s.ServerNames[0] != "localhost" || s.ServerNames[1] != "" { + t.Errorf("server_names %q", s.ServerNames) + } +} + +// TestStoreStartTLS: StoreOpts.StartTLS {} offers STARTTLS with the program's certificate, and +// the knobs of StartTLSOptions are read. +func TestStoreStartTLS(t *testing.T) { + p := startProc(t, `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {}}}}`) + c := p.dial() + tc, err := c.TryStartTLS(&tls.Config{RootCAs: p.caPool(), ServerName: "localhost"}) + if err != nil { + t.Fatal(err) + } + cmd(t, tc, 502, "STARTTLS") + tc.Close() + s, _ := p.finish() + if s.TLSHandshakes != 1 { + t.Errorf("tls_handshakes %d", s.TLSHandshakes) + } + + p = startProc(t, `{"Mode": "peer", "PeerOptions": {"StartTLS": {"Refuse": "580 no thanks", "HideCapability": true}}}`) + c = p.dial() + var se *client.StartTLSError + if _, err := c.TryStartTLS(&tls.Config{RootCAs: p.caPool()}); !errors.As(err, &se) || se.Code != 580 { + t.Errorf("refused STARTTLS: %v", err) + } + p.finish() +} + +// TestListenFlag: -listen picks the address; Options.Addr is the default. +func TestListenFlag(t *testing.T) { + addr := freeAddr(t) + p := startProc(t, `{"Mode": "store"}`, "-listen", addr) + if p.ready.Addr != addr { + t.Errorf("addr %q, want %q", p.ready.Addr, addr) + } + p.finish() + addr = freeAddr(t) + p = startProc(t, `{"Mode": "store", "Options": {"Addr": "`+addr+`"}}`) + if p.ready.Addr != addr { + t.Errorf("addr %q, want %q", p.ready.Addr, addr) + } + p.finish() + busy := fakeserver.New(t, fakeserver.Options{}) + p = launch(t, `{"Mode": "store"}`, "-listen", busy.Addr()) + if code := <-p.code; code != 1 || !strings.Contains(p.stderr.String(), "listen") { + t.Errorf("address in use: exit %d, stderr %s", code, p.stderr.String()) + } +} + +func freeAddr(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + return ln.Addr().String() +} + +// TestSummaryShape: with no client every key is there, empty lists are [] and not null. +func TestSummaryShape(t *testing.T) { + p := startProc(t, `{"Mode": "store"}`) + _ = p.stdin.Close() + line, err := p.out.ReadString('\n') + if err != nil { + t.Fatal(err) + } + var m map[string]json.RawMessage + if err := json.Unmarshal([]byte(line), &m); err != nil { + t.Fatal(err) + } + want := map[string]string{"lines": "[]", "mismatches": "[]", "articles": "[]", "notes": "[]", "accepted": "0", + "counts": "{}", "tls_handshakes": "0", "tls_failed": "0", "server_names": "[]", + "lines_b64": "[]", "mismatches_b64": "[]", "articles_raw": "[]", "max_open": "0", "closed": "0"} + for k, v := range want { + if string(m[k]) != v { + t.Errorf("%s = %s, want %s", k, m[k], v) + } + } + if len(m) != len(want) { + t.Errorf("keys %v", m) + } + if code := <-p.code; code != 0 { + t.Errorf("exit %d", code) + } +} + +// TestSignal: a cancelled context ends the run like end of file. +func TestSignal(t *testing.T) { + p := startProc(t, `{"Mode": "store"}`) + p.cancel() + if _, err := p.out.ReadString('\n'); err != nil { + t.Fatal(err) + } + if code := <-p.code; code != 0 { + t.Errorf("exit %d", code) + } +} + +// TestBadConfig: a config error is exit 2 with the place in stderr, before anything listens. +func TestBadConfig(t *testing.T) { + cases := []struct{ name, cfg, want string }{ + {"unknown-top-key", `{"Mode": "store", "mode": "peer"}`, `config.mode: unknown key`}, + {"unknown-nested", `{"Mode": "store", "Options": {"Faults": {"Delayy": {}}}}`, `config.Options.Faults.Delayy: unknown key (the keys of Faults are Delay,`}, + {"typo-in-store", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "Bodyy": []}]}}`, `config.Store.Articles[0].Bodyy: unknown key`}, + {"no-mode", `{}`, `config.Mode: want one of store, peer, script, raw`}, + {"bad-mode", `{"Mode": "relay"}`, `"relay"`}, + {"not-json", `{"Mode": `, "unexpected end of JSON input"}, + {"trailing", `{"Mode": "store"} {}`, "invalid character"}, + {"duration-string", `{"Mode": "store", "Options": {"Faults": {"Delay": {"DATE": "2s"}}}}`, `config.Options.Faults.Delay["DATE"]: want a number of milliseconds`}, + {"wrong-type", `{"Mode": "store", "Options": {"Faults": {"CloseAfter": "x"}}}`, "config.Options.Faults.CloseAfter: want int"}, + {"lower-case-command", `{"Mode": "store", "Options": {"Faults": {"Delay": {"date": 2000}}}}`, `config.Options.Faults.Delay["date"]: command names are upper case ("DATE")`}, + {"lower-case-drop", `{"Mode": "store", "Options": {"Faults": {"DropInMultiline": {"Article": 1}}}}`, `DropInMultiline["Article"]`}, + {"both-body-forms", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "Body": ["x"], "BodyB64": ["eA=="]}]}}`, "Articles[0].Body and BodyB64 are both set"}, + {"bad-body-b64", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "BodyB64": ["!"]}]}}`, "Articles[0].BodyB64[0]:"}, + {"bad-headers-b64", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "HeadersB64": ["!!"]}]}}`, "config.Store.Articles[0].HeadersB64[0]:"}, + {"sleep-over-deadline", `{"Mode": "raw", "Raw": [{"Sleep": 30000}]}`, "config.Raw[0].Sleep: 30s is longer than Options.Deadline (10s)"}, + {"gap-over-deadline", `{"Mode": "raw", "Options": {"Deadline": 500}, "Raw": [{"Send": [{"Text": "x", "Gap": 600}]}]}`, "config.Raw[0].Send[0].Gap: 600ms is longer than Options.Deadline (500ms)"}, + {"splitgap-over-deadline", `{"Mode": "raw", "Raw": [{"Send": [{"Text": "xy", "Split": 1, "SplitGap": 10001}]}]}`, "config.Raw[0].Send[0].SplitGap"}, + {"handler", `{"Mode": "store", "Options": {"Handler": {}}}`, "config.Options.Handler: cannot be set from JSON"}, + {"store-object", `{"Mode": "store", "Options": {"Store": {}}}`, "config.Options.Store: cannot be set from JSON"}, + {"peer-in-store", `{"Mode": "store", "PeerOptions": {"User": "u"}}`, `config.PeerOptions is not used in mode "store"`}, + {"faults-in-raw", `{"Mode": "raw", "Options": {"Faults": {"CloseAfter": 1}}}`, `config.Options.Faults is not used in mode "raw"`}, + {"raw-in-store", `{"Mode": "store", "Raw": [{}]}`, `config.Raw is not used in mode "store"`}, + {"bad-regexp", `{"Mode": "raw", "Raw": [{"Expect": "(("}]}`, "config.Raw[0].Expect"}, + {"bad-script-regexp", `{"Mode": "script", "Options": {"Script": [{"Expect": "(("}]}}`, "config.Options.Script[0].Expect"}, + {"bad-then", `{"Mode": "raw", "Raw": [{"Then": "explode"}]}`, `config.Raw[0].Then`}, + {"bad-b64", `{"Mode": "raw", "Raw": [{"Send": [{"B64": "!!"}]}]}`, "config.Raw[0].Send[0]: B64"}, + {"b64-and-text", `{"Mode": "raw", "Raw": [{"Send": [{"B64": "AA==", "Text": "x"}]}]}`, "B64 and Text are both set"}, + {"bad-time", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "Date": "yesterday"}]}}`, "config.Store.Articles[0].Date"}, + {"empty-group", `{"Mode": "store", "Store": {"Groups": [{}]}}`, "Groups[0].Name is empty"}, + {"msgid-differs", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "MessageID": "", "Headers": ["Message-ID: "]}]}}`, "differs from Message-ID header"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := launch(t, tc.cfg) + if code := <-p.code; code != 2 || !strings.Contains(p.stderr.String(), tc.want) { + t.Errorf("exit %d, want 2 with %q\nstderr: %s", code, tc.want, p.stderr.String()) + } + if line, _ := p.out.ReadString('\n'); line != "" { + t.Errorf("stdout %q", line) + } + }) + } +} + +// TestUsage: arguments. +func TestUsage(t *testing.T) { + cases := []struct { + name string + args []string + code int + want string + }{ + {"no-arg", nil, 2, "exactly one argument"}, + {"two-args", []string{"a", "b"}, 2, "exactly one argument"}, + {"missing-file", []string{filepath.Join(t.TempDir(), "nope.json")}, 2, "no such file"}, + {"bad-flag", []string{"-nope"}, 2, "-nope"}, + {"help", []string{"-h"}, 0, "serves until stdin reaches end of file"}, + {"help-ready-keys", []string{"-h"}, 0, "{addr, host, port, ca, ca_pem}"}, + {"help-summary-keys", []string{"-h"}, 0, "server_names, max_open, closed}"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var out, errb bytes.Buffer + if code := run(context.Background(), tc.args, strings.NewReader(""), &out, &errb); code != tc.code { + t.Errorf("exit %d, want %d", code, tc.code) + } + if !strings.Contains(errb.String(), tc.want) || out.Len() != 0 { + t.Errorf("stderr %q, stdout %q, want stderr with %q", errb.String(), out.String(), tc.want) + } + }) + } +} + +// sample returns JSON that decodes into a value of type t, false for a type the decoder cannot +// set. +func sample(t reflect.Type) (string, bool) { + switch t { + case reflect.TypeFor[time.Duration](): + return "1.5", true + case reflect.TypeFor[time.Time](): + return `"2020-01-02T03:04:05Z"`, true + } + switch t.Kind() { + case reflect.Bool: + return "true", true + case reflect.String: + return `"x"`, true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return "1", true + case reflect.Float32, reflect.Float64: + return "1.5", true + case reflect.Map: + v, ok := sample(t.Elem()) + return `{"K": ` + v + `}`, ok && t.Key().Kind() == reflect.String + case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return `"AAE="`, true // []byte is base64 + } + v, ok := sample(t.Elem()) + return `[` + v + `]`, ok + case reflect.Pointer: + if t.Elem() == reflect.TypeFor[tls.Config]() { + return "true", true + } + return sample(t.Elem()) + case reflect.Struct: + if !hasExported(t) { + return "", false + } + var parts []string // every field that can be set, so a nested struct is exercised too + for i := range t.NumField() { + if f := t.Field(i); f.IsExported() { + if v, ok := sample(f.Type); ok { + parts = append(parts, `"`+f.Name+`": `+v) + } + } + } + return `{` + strings.Join(parts, ", ") + `}`, true + } + return "", false +} + +func hasExported(t reflect.Type) bool { + for i := range t.NumField() { + if t.Field(i).IsExported() { + return true + } + } + return false +} + +// TestEveryKitFieldDecodes: every exported field of the kit's option types that is data (not a +// function or a channel) can be set from JSON, so a field added to the kit works without a +// change here; a new field of a shape the decoder cannot read fails this test. +func TestEveryKitFieldDecodes(t *testing.T) { + d := &decoder{tls: &tls.Config{}} + for _, v := range []any{fakeserver.Options{}, fakeserver.StoreOptions{}, fakeserver.PeerOptions{}, fakeserver.Faults{}, fakeserver.StartTLSOptions{}, fakeserver.Step{}} { + typ := reflect.TypeOf(v) + for i := range typ.NumField() { + f := typ.Field(i) + if !f.IsExported() { + continue + } + js, ok := sample(f.Type) + if !ok { + if k := f.Type.Kind(); k != reflect.Func && k != reflect.Chan && f.Name != "Store" { + t.Errorf("%s.%s (%s): not settable from JSON, and not a function or channel", typ.Name(), f.Name, f.Type) + } + continue + } + ptr := reflect.New(typ) + if err := d.decode(ptr.Elem(), json.RawMessage(`{"`+f.Name+`": `+js+`}`), typ.Name()); err != nil { + t.Errorf("%s.%s: %v", typ.Name(), f.Name, err) + continue + } + if f.Type.Kind() != reflect.Struct && ptr.Elem().Field(i).IsZero() { + t.Errorf("%s.%s stayed zero", typ.Name(), f.Name) + } + } + } +} + +// TestDurations: milliseconds, fractions allowed. +func TestDurations(t *testing.T) { + var f fakeserver.Faults + d := &decoder{} + if err := d.decode(reflect.ValueOf(&f).Elem(), json.RawMessage(`{"Delay": {"A": 2500, "B": 0.5}, "Drip": {"C": 1}}`), "f"); err != nil { + t.Fatal(err) + } + if f.Delay["A"] != 2500*time.Millisecond || f.Delay["B"] != 500*time.Microsecond || f.Drip["C"] != time.Millisecond { + t.Errorf("%v %v", f.Delay, f.Drip) + } +} + +func b64(t *testing.T, s string) string { + t.Helper() + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// TestBinarySafe: bytes that are not UTF-8 come back exactly in lines_b64 and articles_raw (the +// plain fields hold U+FFFD), and a Store serves them from HeadersB64 and BodyB64. +func TestBinarySafe(t *testing.T) { + const art = "Subject: caf\xe9\r\n\r\n\xff\xfe\r\n..dot\r\n" + p := startProc(t, `{"Mode": "peer"}`) + c := p.dial() + cmd(t, c, 335, "IHAVE ") + if err := c.Send(art + ".\r\n"); err != nil { + t.Fatal(err) + } + if code, _, err := c.ReadStatus(); err != nil || code != 235 { + t.Fatalf("%d %v", code, err) + } + c.Send("XYZ caf\xe9\r\n") + cmd(t, c, 500, "DATE") + s, _ := p.finish() + if len(s.ArticlesRaw) != 1 || b64(t, s.ArticlesRaw[0]) != art { + t.Errorf("articles_raw %q", s.ArticlesRaw) + } + if !strings.Contains(s.Articles[0][0], "\ufffd") { + t.Errorf("the plain form is lossy by nature: %q", s.Articles[0]) + } + if len(s.LinesB64) != len(s.Lines) || b64(t, s.LinesB64[1]) != "XYZ caf\xe9" { + t.Errorf("lines_b64 %q lines %q", s.LinesB64, s.Lines) + } + + // Store: bytes in, bytes out. + p = startProc(t, `{"Mode": "store", "Store": {"Articles": [{"Group": "a.b", "Number": 1, "MessageID": "", + "HeadersB64": ["`+base64.StdEncoding.EncodeToString([]byte("Subject: caf\xe9"))+`"], + "BodyB64": ["`+base64.StdEncoding.EncodeToString([]byte("\xff\xfe"))+`"]}]}}`) + c = p.dial() + cmd(t, c, 222, "BODY ") + lines, err := c.TryReadDotLines() + if err != nil || len(lines) != 1 || lines[0] != "\xff\xfe" { + t.Errorf("BODY: %q %v", lines, err) + } + cmd(t, c, 221, "HEAD ") + lines, _ = c.TryReadDotLines() + if !slices.Contains(lines, "Subject: caf\xe9") { + t.Errorf("HEAD: %q", lines) + } + p.finish() + + // Raw mode: the dot block as the bytes were sent. + p = startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, {"Expect": "POST", "ReadDot": true, "Send": [{"Text": "240 ok\r\n"}]}]}`) + c = p.dial() + c.Send("POST\r\n\xe9\xff\r\n.\r\n") + if code, _, err := c.ReadStatus(); err != nil || code != 240 { + t.Fatalf("%d %v", code, err) + } + s, _ = p.finish() + if len(s.ArticlesRaw) != 1 || b64(t, s.ArticlesRaw[0]) != "\xe9\xff\r\n" { + t.Errorf("raw articles_raw %q", s.ArticlesRaw) + } +} + +// TestRawBoundedByDeadline: a raw step that sleeps for a minute ends with the server, and a +// client that says nothing is let go after the Deadline. +func TestRawBoundedByDeadline(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Options": {"Deadline": 600000}, "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, + {"Expect": "X", "Sleep": 60000, "Send": [{"Text": "1 a", "Gap": 60000, "Split": 1, "SplitGap": 60000}]}]}`) + c := p.dial() + c.Send("X\r\n") + time.Sleep(100 * time.Millisecond) + start := time.Now() + p.finish() + if d := time.Since(start); d > 3*time.Second { + t.Errorf("shutdown took %v while a step slept", d) + } + + p = startProc(t, `{"Mode": "raw", "Options": {"Deadline": 200}, "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, {"Expect": "X"}]}`) + c = p.dial() + if _, err := c.ReadLine(); !errors.Is(err, io.EOF) { + t.Errorf("a silent client was not let go: %v", err) + } + s, _ := p.finish() + if len(s.Notes) != 1 || !strings.Contains(s.Notes[0], "within the Deadline") { + t.Errorf("notes %q", s.Notes) + } +} + +// TestRawLineLimit: a line over 1 MiB closes the connection and is noted, not held in memory. +func TestRawLineLimit(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, {"Expect": "X"}]}`) + c := p.dial() + go c.Send(strings.Repeat("a", maxLine+1<<16)) // the write may fail when the server closes + if _, err := c.ReadLine(); err == nil { + t.Error("the connection stayed open") + } + s, _ := p.finish() + if len(s.Notes) != 1 || !strings.Contains(s.Notes[0], "a line over") || len(s.Lines) != 0 { + t.Errorf("notes %q lines %d", s.Notes, len(s.Lines)) + } +} + +// TestEmptySections: sections that a caller writes empty are fine in any mode. +func TestEmptySections(t *testing.T) { + for _, cfg := range []string{ + `{"Mode": "raw", "Options": {"Faults": {"Delay": {}}}}`, + `{"Mode": "store", "Raw": [], "PeerOptions": {"Have": {}}, "Options": {"Script": []}}`, + `{"Mode": "peer", "Store": {"Groups": [], "Articles": []}, "Options": {"StoreOpts": {"OverviewFmt": []}}}`, + `{"Mode": "store", "Options": {"Faults": {"OverrideLine": {"article 2": ["430 no"]}}}}`, // the kit normalises + } { + if _, err := parseConfig([]byte(cfg), &tls.Config{}); err != nil { + t.Errorf("%s: %v", cfg, err) + } + } + for _, cfg := range []string{ + `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {}}}, "Raw": [{}]}`, + `{"Mode": "raw", "Options": {"StoreOpts": {"StartTLS": {}}}}`, // a StartTLS object is not empty + } { + if _, err := parseConfig([]byte(cfg), &tls.Config{}); err == nil { + t.Errorf("%s: accepted", cfg) + } + } +} + +// TestStartTLSConfigOff: StartTLS {} offers STARTTLS, "Config": false does not, and the +// certificate of the program is the default only when Config is not named. +func TestStartTLSConfigOff(t *testing.T) { + for cfg, want := range map[string]int{ + `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {}}}}`: 382, + `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {"Config": true}}}}`: 382, + `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {"Config": false}}}}`: 500, + `{"Mode": "peer", "PeerOptions": {"StartTLS": {"Config": false, "Refuse": "502"}}}`: 500, + } { + p := startProc(t, cfg) + c := p.dial() + if code, line, err := c.TryCmd("STARTTLS"); err != nil || code != want { + t.Errorf("%s: STARTTLS %d %q %v, want %d", cfg, code, line, err, want) + } + p.finish() + } +} + +// TestCounters: max_open and closed, and ca_pem in the ready line. +func TestCounters(t *testing.T) { + p := startProc(t, `{"Mode": "store"}`) + pem, err := os.ReadFile(p.ready.CA) + if err != nil || p.ready.CAPEM == "" || p.ready.CAPEM != string(pem) { + t.Errorf("ca_pem %q, file %q, %v", p.ready.CAPEM, pem, err) + } + a, b := p.dial(), p.dial() + cmd(t, a, 111, "DATE") + cmd(t, b, 111, "DATE") + a.Close() + b.Close() + s, _ := p.finish() + if s.MaxOpen != 2 || s.Closed != 2 || s.Accepted != 2 { + t.Errorf("max_open %d closed %d accepted %d", s.MaxOpen, s.Closed, s.Accepted) + } +} + +// TestIsEmpty: what counts as an empty section. +func TestIsEmpty(t *testing.T) { + for _, tc := range []struct { + v any + want bool + }{ + {fakeserver.Faults{}, true}, + {fakeserver.Faults{Delay: map[string]time.Duration{}}, true}, + {fakeserver.Faults{Delay: map[string]time.Duration{"A": 0}}, false}, + {fakeserver.StoreOptions{StartTLS: &fakeserver.StartTLSOptions{}}, false}, + {[]rawStep{}, true}, + {[]rawStep{{}}, false}, + } { + if got := isEmpty(reflect.ValueOf(tc.v)); got != tc.want { + t.Errorf("isEmpty(%#v) = %v", tc.v, got) + } + } +} + +// TestBytesAreBase64: a []byte field takes a base64 string, as in encoding/json. +func TestBytesAreBase64(t *testing.T) { + var v struct{ B []byte } + d := &decoder{} + if err := d.decode(reflect.ValueOf(&v).Elem(), json.RawMessage(`{"B": "AAEC"}`), "v"); err != nil || !bytes.Equal(v.B, []byte{0, 1, 2}) { + t.Errorf("%v %v", v.B, err) + } + if err := d.decode(reflect.ValueOf(&v).Elem(), json.RawMessage(`{"B": [0, 1]}`), "v"); err == nil { + t.Error("an array was taken for bytes") + } +} + +// TestMain lets TestSignal run the real main in a child process. +func TestMain(m *testing.M) { + if cfg := os.Getenv("NNTP_FAKESERVER_TEST_CONFIG"); cfg != "" { + os.Args = []string{"nntp-fakeserver", cfg} + main() + return + } + os.Exit(m.Run()) +} + +// TestMainSignal: the real main, as a process: SIGTERM ends the server with its summary and exit +// status 0, and so does EOF on stdin. +func TestMainSignal(t *testing.T) { + for _, how := range []string{"SIGTERM", "SIGINT", "EOF"} { + t.Run(how, func(t *testing.T) { + cmd := exec.Command(os.Args[0]) + cmd.Env = append(os.Environ(), "NNTP_FAKESERVER_TEST_CONFIG="+writeConfig(t, `{"Mode": "store", "Options": {"RawStall": true}}`)) + in, _ := cmd.StdinPipe() + out, _ := cmd.StdoutPipe() + var errb bytes.Buffer + cmd.Stderr = &errb + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + br := bufio.NewReader(out) + if _, err := br.ReadString('\n'); err != nil { + t.Fatalf("%v: %s", err, errb.String()) + } + switch how { + case "SIGTERM": + cmd.Process.Signal(syscall.SIGTERM) + case "SIGINT": + cmd.Process.Signal(syscall.SIGINT) + default: + in.Close() + } + line, err := br.ReadString('\n') + if err != nil || !strings.Contains(line, `"accepted":0`) { + t.Errorf("summary %q %v", line, err) + } + if err := cmd.Wait(); err != nil { + t.Errorf("exit: %v %s", err, errb.String()) + } + }) + } +} + +// TestByteFaults: the byte-level Faults (Split, Trailing, DropAfterBytes, Abort, OverrideFinal, +// DelayFinal, DropInUpload) come through with their nested fields and durations in ms. +func TestByteFaults(t *testing.T) { + cfg, err := parseConfig([]byte(`{"Mode": "store", "Options": {"Faults": { + "Split": {"DATE": {"At": [1, 5], "Size": 3, "Gap": 2.5}}, + "Trailing": {"DATE": {"Lines": ["111 x", "."], "Gap": 40}}, + "DropAfterBytes": {"GROUP": 4}, "Abort": {"GROUP": true}, + "OverrideFinal": {"POST": ["441 no"]}, "DelayFinal": {"POST": 15}, "DropInUpload": {"POST": 2}}}}`), &tls.Config{}) + if err != nil { + t.Fatal(err) + } + f := cfg.Options.Faults + if sp := f.Split["DATE"]; !slices.Equal(sp.At, []int{1, 5}) || sp.Size != 3 || sp.Gap != 2500*time.Microsecond { + t.Errorf("Split %+v", sp) + } + if tr := f.Trailing["DATE"]; !slices.Equal(tr.Lines, []string{"111 x", "."}) || tr.Gap != 40*time.Millisecond { + t.Errorf("Trailing %+v", tr) + } + if f.DropAfterBytes["GROUP"] != 4 || !f.Abort["GROUP"] || f.OverrideFinal["POST"][0] != "441 no" || + f.DelayFinal["POST"] != 15*time.Millisecond || f.DropInUpload["POST"] != 2 { + t.Errorf("faults %+v", f) + } + + // End to end: the status line of DATE in two writes with a pause, then the trailing line. + p := startProc(t, `{"Mode": "store", "Options": {"Faults": { + "Split": {"DATE": {"At": [3], "Gap": 100}}, "Trailing": {"DATE": {"Lines": ["TRAILER"], "Gap": 50}}}}}`) + nc, err := net.DialTimeout("tcp", p.ready.Addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + br := bufio.NewReader(nc) + br.ReadString('\n') + io.WriteString(nc, "DATE\r\n") + _ = nc.SetReadDeadline(time.Now().Add(5 * time.Second)) + var got []string + for range 2 { + l, err := br.ReadString('\n') + if err != nil { + t.Fatal(err) + } + got = append(got, strings.TrimSpace(l)) + } + if !strings.HasPrefix(got[0], "111 ") || got[1] != "TRAILER" { + t.Errorf("got %q", got) + } + p.finish() +} diff --git a/cmd/nntp-fakeserver/raw.go b/cmd/nntp-fakeserver/raw.go new file mode 100644 index 0000000..1550cf8 --- /dev/null +++ b/cmd/nntp-fakeserver/raw.go @@ -0,0 +1,286 @@ +package main + +import ( + "bufio" + "compress/flate" + "crypto/tls" + "errors" + "fmt" + "net" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/go-while/nntp-testkit/fakeserver" +) + +// recorder holds what raw mode saw (the fakeserver records the other modes; a raw step reads +// the socket itself, so the fakeserver sees no lines). +type recorder struct { + mu sync.Mutex + lines []string + mismatches []string + articles [][]string + articleRaw [][]byte // the blocks as read, without the "." line (articles_raw) + notes []string + counts map[string]int +} + +func (r *recorder) line(s string) { + r.mu.Lock() + defer r.mu.Unlock() + r.lines = append(r.lines, s) + if r.counts == nil { + r.counts = make(map[string]int) + } + var cmd string + if f := strings.Fields(s); len(f) > 0 { + cmd = strings.ToUpper(f[0]) + } + r.counts[cmd]++ +} + +func (r *recorder) mismatch(s string) { + r.mu.Lock() + defer r.mu.Unlock() + r.mismatches = append(r.mismatches, s) +} + +func (r *recorder) note(format string, a ...any) { + r.mu.Lock() + defer r.mu.Unlock() + r.notes = append(r.notes, fmt.Sprintf(format, a...)) +} + +func (r *recorder) article(block []string, wire []byte) { + r.mu.Lock() + defer r.mu.Unlock() + r.articles = append(r.articles, block) + r.articleRaw = append(r.articleRaw, wire) +} + +// The limits of what a raw connection reads (a client or a DEFLATE bomb cannot fill the memory): +// one line, one dot block, and everything read on one connection. A connection that goes over +// one is closed and a note says so. +const ( + maxLine = 1 << 20 + maxBlock = 16 << 20 + maxConn = 64 << 20 +) + +var errTooLong = errors.New("line longer than the limit") + +// rawConn is one connection of mode raw. +type rawConn struct { + c *fakeserver.Conn + rec *recorder + deadline time.Duration // Options.Deadline: bounds every wait, read and write + conn net.Conn + r *bufio.Reader + total int // bytes read so far +} + +func (rc *rawConn) tcp() *net.TCPConn { + nc := rc.conn + if t, ok := nc.(*tls.Conn); ok { + nc = t.NetConn() + } + t, _ := nc.(*net.TCPConn) + return t +} + +// readWire reads one line with its line end. A timed read waits at most the Deadline for it +// (the read of a line the script expects); an untimed one waits for the client (hang, the lines +// after the script) and ends with the connection. A line over maxLine or a connection over +// maxConn ends the read with a note. +func (rc *rawConn) readWire(timed bool) ([]byte, error) { + if timed { + _ = rc.conn.SetReadDeadline(time.Now().Add(rc.deadline)) + } else { + _ = rc.conn.SetReadDeadline(time.Time{}) + } + var line []byte + for { + part, err := rc.r.ReadSlice('\n') + line = append(line, part...) + rc.total += len(part) + switch { + case err == nil: + if rc.total > maxConn { + rc.rec.note("read: over %d bytes on one connection, closed", maxConn) + return line, errTooLong + } + return line, nil + case errors.Is(err, bufio.ErrBufferFull): + case errors.Is(err, os.ErrDeadlineExceeded): + rc.rec.note("read: nothing from the client within the Deadline (%v)", rc.deadline) + return line, err + default: + var ce flate.CorruptInputError + if errors.As(err, &ce) { + rc.rec.note("read: %v", err) + } + return line, err + } + if len(line) > maxLine { + rc.rec.note("read: a line over %d bytes, connection closed", maxLine) + return line, errTooLong + } + if rc.total > maxConn { + rc.rec.note("read: over %d bytes on one connection, closed", maxConn) + return line, errTooLong + } + } +} + +// readLine is readWire without the line end. +func (rc *rawConn) readLine(timed bool) (string, error) { + w, err := rc.readWire(timed) + if err != nil { + return "", err + } + return strings.TrimRight(string(w), "\r\n"), nil +} + +// write writes data within the Deadline. +func (rc *rawConn) write(data []byte) error { + _ = rc.conn.SetWriteDeadline(time.Now().Add(rc.deadline)) + _, err := rc.conn.Write(data) + return err +} + +// writeChunk writes the chunk after its Gap, in one write or in pieces of Split bytes with +// SplitGap between them. The pauses end when the server closes or at the Deadline. +func (rc *rawConn) writeChunk(ch rawChunk) error { + data, err := ch.bytes() + if err != nil { + return err + } + if ch.Gap > 0 && !rc.c.Sleep(ch.Gap) { + return errors.New("server closed") + } + if ch.Split <= 0 { + return rc.write(data) + } + for i := 0; i < len(data); i += ch.Split { + if i > 0 && ch.SplitGap > 0 && !rc.c.Sleep(ch.SplitGap) { + return errors.New("server closed") + } + if err := rc.write(data[i:min(i+ch.Split, len(data))]); err != nil { + return err + } + } + return nil +} + +// rawHandler runs the steps on every connection, on Conn.Raw(). After the last step it records +// every line and answers it with a 500, so a client that sends more than the script expects +// shows up in mismatches. +func rawHandler(steps []rawStep, rec *recorder, tlsCfg *tls.Config, deadline time.Duration) func(*fakeserver.Conn) { + res := make([]*regexp.Regexp, len(steps)) + for i, st := range steps { + if st.Expect != nil { + res[i] = regexp.MustCompile("^(?:" + *st.Expect + ")$") // checked by parseConfig + } + } + return func(c *fakeserver.Conn) { + rc := &rawConn{c: c, rec: rec, deadline: deadline, conn: c.Raw()} + if t := rc.tcp(); t != nil { + _ = t.SetNoDelay(true) + } + rc.r = bufio.NewReader(rc.conn) + for i, st := range steps { + if res[i] != nil { + for { + line, err := rc.readLine(true) + if err != nil { + return + } + rec.line(line) + if res[i].MatchString(line) { + break + } + rec.mismatch(line) + _ = rc.write([]byte("500 unexpected " + line + "\r\n")) + } + } + if st.ReadDot { + var block []string + var wire []byte + for { + w, err := rc.readWire(true) + if err != nil { + if !errors.Is(err, errTooLong) && !errors.Is(err, os.ErrDeadlineExceeded) { + rec.note("dot block read: %v after %d lines", err, len(block)) + } + return + } + line := strings.TrimRight(string(w), "\r\n") + block = append(block, line) + if line != "." { + wire = append(wire, w...) + } + if len(wire) > maxBlock { + rec.note("dot block: over %d bytes, connection closed", maxBlock) + return + } + if line == "." || (st.DotMax > 0 && len(block) >= st.DotMax) { + break + } + } + rec.article(block, wire) + } + for _, ch := range st.Send { + if err := rc.writeChunk(ch); err != nil { + rec.note("write: %v", err) + return + } + } + if st.Sleep > 0 && !c.Sleep(st.Sleep) { + return + } + switch st.Then { + case "close": + return + case "rst": + if t := rc.tcp(); t != nil { + _ = t.SetLinger(0) + } + return + case "hang": + for { // holds the connection: no read deadline + line, err := rc.readLine(false) + if err != nil { + return + } + rec.line(line) + } + case "starttls": + if n := rc.r.Buffered(); n > 0 { + rec.note("starttls: %d bytes of plaintext pipelined after the STARTTLS line", n) + return + } + // The handshake is bounded by the server's Deadline; the server counts it. + if err := c.StartTLS(tlsCfg); err != nil { + rec.note("starttls handshake: %v", err) + return + } + rc.conn = c.Raw() + rc.r = bufio.NewReader(rc.conn) + case "inflate": + rc.r = bufio.NewReader(flate.NewReader(rc.r)) + } + } + for { // the lines after the script: no read deadline either + line, err := rc.readLine(false) + if err != nil { + return + } + rec.line(line) + rec.mismatch(line) + _ = rc.write([]byte("500 unexpected " + line + "\r\n")) + } + } +}