From 0099507ac51c08fbdb7f8510749d0d1b32a38f97 Mon Sep 17 00:00:00 2001 From: go-while Date: Thu, 12 Nov 2026 13:18:54 +0000 Subject: [PATCH 1/4] cmd/nntp-fakeserver: the fake server as a process, configured by JSON A standalone program for test suites in other languages (Python, shell, C): it reads a JSON config, prints {addr, host, port, ca} when it listens, serves until stdin reaches end of file, then prints a summary {lines, mismatches, articles, notes, accepted, counts, tls_handshakes, tls_failed, server_names}. Modes: store, peer, script (the kit's) and raw (a byte-exact script with split writes, RST, STARTTLS and inflate, generalised from pynntp's fault server). The config keys are the Go field names of fakeserver.Options, StoreOptions, PeerOptions, Faults and StartTLSOptions, durations in milliseconds; the decoder walks the types, so a field added to the kit works from JSON at once, and an unknown key is an error. The certificate is tlscert.GenerateSelfSigned, written to a temp dir that is removed on exit. Co-Authored-By: Claude Opus 5.5 --- cmd/nntp-fakeserver/config.go | 370 ++++++++++++++++ cmd/nntp-fakeserver/main.go | 276 ++++++++++++ cmd/nntp-fakeserver/main_test.go | 735 +++++++++++++++++++++++++++++++ cmd/nntp-fakeserver/raw.go | 197 +++++++++ 4 files changed, 1578 insertions(+) create mode 100644 cmd/nntp-fakeserver/config.go create mode 100644 cmd/nntp-fakeserver/main.go create mode 100644 cmd/nntp-fakeserver/main_test.go create mode 100644 cmd/nntp-fakeserver/raw.go diff --git a/cmd/nntp-fakeserver/config.go b/cmd/nntp-fakeserver/config.go new file mode 100644 index 0000000..0ddc465 --- /dev/null +++ b/cmd/nntp-fakeserver/config.go @@ -0,0 +1,370 @@ +package main + +import ( + "bytes" + "crypto/tls" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "math" + "reflect" + "regexp" + "slices" + "sort" + "strings" + "time" + + "github.com/go-while/nntp-testkit/fakeserver" +) + +// config is the JSON document the program reads. Every key is the Go field name of the kit type +// (or of the types below), matched exactly; an unknown key is an error. A field of a kit type +// that is added later can be used from JSON at once, because the decoder walks the types +// (decodeValue), it does not mirror them. +type config struct { + Mode string // "store", "peer", "script" or "raw" + + // Options is fakeserver.Options: Addr, Greeting, NoGreeting, RawStall, TLS (true: implicit + // TLS with the program's certificate), Script (mode script), StoreOpts (mode store), Faults, + // Deadline (ms), MaxConns, MaxConnsGreeting. Store and Handler are not settable. + Options fakeserver.Options + + Store storeConfig // mode store: what the store holds + PeerOptions fakeserver.PeerOptions // mode peer + Raw []rawStep // mode raw: the byte script +} + +// storeConfig is the content of the Store of mode store. +type storeConfig struct { + Groups []groupConfig + Articles []articleConfig +} + +// groupConfig is one group: Created defaults to 2020-01-01 UTC. +type groupConfig struct { + Name, Description string + Created time.Time // RFC 3339 + NoPosting bool // LIST ACTIVE "n" +} + +// articleConfig is one fakeserver.Article filed in Group; a zero Date and Arrived default to +// 2020-01-02T03:04:05Z (so that replies are the same on every run). +type articleConfig struct { + Group string + Number int64 + MessageID string + Headers []string + Body []string + Date time.Time // RFC 3339 + Arrived time.Time +} + +// rawStep is one step of mode raw: read one client line (unless Expect is null or absent) that +// matches Expect, then the dot block when ReadDot, write the chunks of Send, wait Sleep, then +// act as Then says. +type rawStep struct { + Expect *string // anchored regexp for one client line; null: read nothing + ReadDot bool // after the line, read a dot-terminated block (recorded in articles) + DotMax int // >0 with ReadDot: read only this many lines, then go on + Send []rawChunk // each chunk is its own write + Sleep time.Duration // ms, after sending, before Then + Then string // "", close, rst, hang, starttls or inflate +} + +// rawChunk is bytes to write: B64 (base64) or Text (UTF-8, JSON escapes give CR, LF and so on). +type rawChunk struct { + B64 string + Text string + Gap time.Duration // ms, pause before the chunk + Split int // >0: write in pieces of this many bytes + SplitGap time.Duration // ms, pause between pieces +} + +// thens are the actions after a raw step. +var thens = []string{"", "close", "rst", "hang", "starttls", "inflate"} + +var modes = []string{"store", "peer", "script", "raw"} + +// parseConfig decodes data and checks that the parts fit the mode. tlsCfg is the server config +// that "TLS": true and an empty StartTLS stand for. +func parseConfig(data []byte, tlsCfg *tls.Config) (*config, error) { + var raw json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + var cfg config + d := decoder{tls: tlsCfg} + if err := d.decode(reflect.ValueOf(&cfg).Elem(), raw, "config"); err != nil { + return nil, err + } + if !slices.Contains(modes, cfg.Mode) { + return nil, fmt.Errorf(`config.Mode: want one of %s, got %q`, strings.Join(modes, ", "), cfg.Mode) + } + if err := cfg.check(); err != nil { + return nil, err + } + for _, p := range []*fakeserver.StartTLSOptions{cfg.Options.StoreOpts.StartTLS, cfg.PeerOptions.StartTLS} { + if p != nil && p.Config == nil { + p.Config = tlsCfg + } + } + return &cfg, nil +} + +// check rejects the parts of the config that the mode does not use (a setting that is silently +// ignored is a typo that passes), and bad regexps, base64 and thens in raw mode. +func (cfg *config) check() error { + o := reflect.ValueOf(&cfg.Options).Elem() + parts := []struct { + name string + v reflect.Value + in string // the modes that use it + }{ + {"Options.StoreOpts", o.FieldByName("StoreOpts"), "store"}, + {"Store", reflect.ValueOf(&cfg.Store).Elem(), "store"}, + {"PeerOptions", reflect.ValueOf(&cfg.PeerOptions).Elem(), "peer"}, + {"Options.Script", o.FieldByName("Script"), "script"}, + {"Options.Faults", o.FieldByName("Faults"), "store, peer, script"}, + {"Options.Greeting", o.FieldByName("Greeting"), "store, peer, script"}, + {"Raw", reflect.ValueOf(&cfg.Raw).Elem(), "raw"}, + } + for _, p := range parts { + if !p.v.IsZero() && !strings.Contains(p.in, cfg.Mode) { + return fmt.Errorf("config.%s is not used in mode %q (only in %s)", p.name, cfg.Mode, p.in) + } + } + for i, st := range cfg.Options.Script { + if _, err := regexp.Compile("^(?:" + st.Expect + ")$"); err != nil { + return fmt.Errorf("config.Options.Script[%d].Expect: %w", i, err) + } + } + if cfg.Mode != "raw" { + return nil + } + for i, st := range cfg.Raw { + if st.Expect != nil { + if _, err := regexp.Compile("^(?:" + *st.Expect + ")$"); err != nil { + return fmt.Errorf("config.Raw[%d].Expect: %w", i, err) + } + } + if !slices.Contains(thens, st.Then) { + return fmt.Errorf("config.Raw[%d].Then: want one of %q, got %q", i, thens, st.Then) + } + for j, ch := range st.Send { + if _, err := ch.bytes(); err != nil { + return fmt.Errorf("config.Raw[%d].Send[%d]: %w", i, j, err) + } + } + } + return nil +} + +// bytes returns the data of the chunk. +func (ch rawChunk) bytes() ([]byte, error) { + if ch.B64 != "" && ch.Text != "" { + return nil, fmt.Errorf("B64 and Text are both set") + } + if ch.Text != "" { + return []byte(ch.Text), nil + } + b, err := base64.StdEncoding.DecodeString(ch.B64) + if err != nil { + return nil, fmt.Errorf("B64: %w", err) + } + return b, nil +} + +// buildStore fills a Store from the config. AddArticle panics on a setup error (a number taken, +// a Message-ID that differs from the header): that becomes an error. +func buildStore(sc storeConfig) (st *fakeserver.Store, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("config.Store: %v", r) + } + }() + day := func(h, m, s int, d int) time.Time { return time.Date(2020, 1, d, h, m, s, 0, time.UTC) } + st = fakeserver.NewStore() + for i, g := range sc.Groups { + if g.Name == "" { + return nil, fmt.Errorf("config.Store.Groups[%d].Name is empty", i) + } + created := g.Created + if created.IsZero() { + created = day(0, 0, 0, 1) + } + st.AddGroupAt(g.Name, g.Description, created) + if g.NoPosting { + st.SetPosting(g.Name, false) + } + } + for i, a := range sc.Articles { + if a.Group == "" { + return nil, fmt.Errorf("config.Store.Articles[%d].Group is empty", i) + } + art := fakeserver.Article{Number: a.Number, MessageID: a.MessageID, Headers: a.Headers, Body: a.Body, Date: a.Date, Arrived: a.Arrived} + if art.Date.IsZero() { + art.Date = day(3, 4, 5, 2) + } + if art.Arrived.IsZero() { + art.Arrived = day(3, 4, 5, 2) + } + st.AddArticle(a.Group, art) + } + return st, nil +} + +// decoder fills Go values from JSON by walking their types. +// +// The rules: a struct is a JSON object whose keys are exported field names (exact match; an +// unknown key is an error), a time.Duration is a number of milliseconds (fractions allowed), a +// map has string keys, a slice is an array, scalars are JSON scalars, a type with an +// UnmarshalJSON (time.Time: RFC 3339) uses it, a *tls.Config is true (the program's server +// config) or false. Functions, channels, interfaces and structs without exported fields (Store) +// cannot be set: naming one is an error. JSON null leaves the zero value. +type decoder struct { + tls *tls.Config +} + +var ( + durationType = reflect.TypeFor[time.Duration]() + tlsConfigType = reflect.TypeFor[tls.Config]() + unmarshalerType = reflect.TypeFor[json.Unmarshaler]() +) + +func isNull(raw json.RawMessage) bool { return bytes.Equal(bytes.TrimSpace(raw), []byte("null")) } + +// pathError is an error at a place in the document. +type pathError struct { + path string + err error +} + +func (e *pathError) Error() string { return e.path + ": " + e.err.Error() } +func (e *pathError) Unwrap() error { return e.err } + +// decode fills dst from raw; an error names the innermost path. +func (d *decoder) decode(dst reflect.Value, raw json.RawMessage, path string) error { + err := d.decodeValue(dst, raw, path) + var pe *pathError + if err != nil && !errors.As(err, &pe) { + return &pathError{path, err} + } + return err +} + +func (d *decoder) decodeValue(dst reflect.Value, raw json.RawMessage, path string) error { + t := dst.Type() + switch { + case t == durationType: + var ms float64 + if err := json.Unmarshal(raw, &ms); err != nil { + return fmt.Errorf("want a number of milliseconds") + } + if math.Abs(ms) > float64(math.MaxInt64/int64(time.Millisecond)) { + return fmt.Errorf("%v ms is out of range", ms) + } + dst.SetInt(int64(math.Round(ms * float64(time.Millisecond)))) + return nil + case t.Kind() == reflect.Pointer && t.Elem() == tlsConfigType: + var on bool + if isNull(raw) { + return nil + } + if err := json.Unmarshal(raw, &on); err != nil { + return fmt.Errorf("want true (the server certificate of this program) or false") + } + if on { + dst.Set(reflect.ValueOf(d.tls)) + } + return nil + case isNull(raw): + return nil + case t.Kind() != reflect.Pointer && reflect.PointerTo(t).Implements(unmarshalerType): + return json.Unmarshal(raw, dst.Addr().Interface()) + } + switch t.Kind() { + case reflect.Pointer: + v := reflect.New(t.Elem()) + if err := d.decodeValue(v.Elem(), raw, path); err != nil { + return err + } + dst.Set(v) + case reflect.Struct: + return d.decodeStruct(dst, raw, path) + case reflect.Map: + if t.Key().Kind() != reflect.String { + return fmt.Errorf("a map with %s keys cannot be set from JSON", t.Key()) + } + var m map[string]json.RawMessage + if err := json.Unmarshal(raw, &m); err != nil { + return fmt.Errorf("want an object") + } + out := reflect.MakeMapWithSize(t, len(m)) + for _, k := range sortedKeys(m) { + e := reflect.New(t.Elem()).Elem() + if err := d.decode(e, m[k], fmt.Sprintf("%s[%q]", path, k)); err != nil { + return err + } + out.SetMapIndex(reflect.ValueOf(k).Convert(t.Key()), e) + } + dst.Set(out) + case reflect.Slice: + var arr []json.RawMessage + if err := json.Unmarshal(raw, &arr); err != nil { + return fmt.Errorf("want an array") + } + out := reflect.MakeSlice(t, len(arr), len(arr)) + for i, r := range arr { + if err := d.decode(out.Index(i), r, fmt.Sprintf("%s[%d]", path, i)); err != nil { + return err + } + } + dst.Set(out) + case reflect.Bool, reflect.String, reflect.Float32, reflect.Float64, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if err := json.Unmarshal(raw, dst.Addr().Interface()); err != nil { + return fmt.Errorf("want %s", t.Kind()) + } + default: + return fmt.Errorf("cannot be set from JSON (type %s)", t) + } + return nil +} + +func (d *decoder) decodeStruct(dst reflect.Value, raw json.RawMessage, path string) error { + t := dst.Type() + var known []string + for i := range t.NumField() { + if t.Field(i).IsExported() { + known = append(known, t.Field(i).Name) + } + } + if len(known) == 0 { + return fmt.Errorf("cannot be set from JSON (type %s)", t) + } + var m map[string]json.RawMessage + if err := json.Unmarshal(raw, &m); err != nil { + return fmt.Errorf("want an object") + } + for _, k := range sortedKeys(m) { + f, ok := t.FieldByName(k) + if !ok || !f.IsExported() || len(f.Index) != 1 { + return &pathError{path + "." + k, fmt.Errorf("unknown key (the keys of %s are %s)", t.Name(), strings.Join(known, ", "))} + } + if err := d.decode(dst.FieldByIndex(f.Index), m[k], path+"."+k); err != nil { + return err + } + } + return nil +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/cmd/nntp-fakeserver/main.go b/cmd/nntp-fakeserver/main.go new file mode 100644 index 0000000..36f9206 --- /dev/null +++ b/cmd/nntp-fakeserver/main.go @@ -0,0 +1,276 @@ +// Command nntp-fakeserver runs the kit's fake NNTP server (package fakeserver) as a process that +// a JSON file configures, so test suites in other languages (Python, shell, C) can use it. +// +// nntp-fakeserver [-listen host:port] config.json +// +// Protocol with the caller: once the server listens, one line of JSON goes to stdout, +// +// {"addr":"127.0.0.1:41233","host":"127.0.0.1","port":41233,"ca":"/tmp/nntp-fakeserver-1/ca.pem"} +// +// ca is a PEM file that holds the certificate of the server (it is self-signed, valid for +// localhost, 127.0.0.1 and ::1): a client verifies implicit TLS and STARTTLS against it. The +// server then runs until stdin reaches end of file (or the process gets SIGINT or SIGTERM). It +// prints one more line of JSON, the summary, removes its temporary directory and exits 0: +// +// {"lines":[...],"mismatches":[...],"articles":[[...]],"notes":[...],"accepted":1, +// "counts":{"GROUP":1},"tls_handshakes":0,"tls_failed":0,"server_names":[]} +// +// lines are the client lines of all connections, mismatches those that did not fit (raw mode, +// script mode), articles the dot blocks received (POST, IHAVE, TAKETHIS, raw ReadDot), notes +// what the server itself ran into (a failed handshake, a cut-off block), counts the number of +// lines per upper-cased command, tls_handshakes and tls_failed the TLS handshakes (implicit and +// STARTTLS) and server_names the SNI of every ClientHello ("" when none). Exit status 2 and a +// message on stderr for a bad config or flag, 1 for another failure. +// +// The config is one JSON object. Its keys are the Go field names of the kit types, matched +// exactly, and a key that does not exist is an error (a typo fails at once). Durations are +// numbers of milliseconds (fractions allowed). Mode says what runs: +// +// store Store + Options.StoreOpts + Options.Faults: the RFC 3977 reader of the kit +// peer PeerOptions: a transit peer (IHAVE, CHECK, TAKETHIS), Options.Faults apply to it +// script Options.Script: lines of text, as fakeserver.Step +// raw Raw: a byte-exact script, run by this program +// +// Common to all modes is Options (fakeserver.Options): Addr, Greeting, NoGreeting, RawStall, +// TLS (true: implicit TLS from the first byte, with the certificate behind ca), Deadline (ms, +// default 10000: bounds every server-side wait and each TLS handshake), MaxConns and +// MaxConnsGreeting. A StartTLS that is {} (or has only the knobs Refuse, HideCapability, +// AuthNeedsTLS, CloseAfterHandshake) offers STARTTLS with the same certificate. Functions, +// channels and the Store itself cannot be set from JSON. A part that the mode does not use is +// an error. +// +// Store mode, with a fault and STARTTLS: +// +// {"Mode": "store", +// "Options": { +// "Greeting": "200 hello", +// "Faults": {"Delay": {"BODY": 2500}, "DropInMultiline": {"ARTICLE": 2}, "LongLine": {"BODY": 65535}}, +// "StoreOpts": {"User": "u", "Pass": "p", "Compress": true, "StartTLS": {"HideCapability": true}}}, +// "Store": { +// "Groups": [{"Name": "alt.test", "Description": "test group"}], +// "Articles": [{"Group": "alt.test", "Number": 1, "MessageID": "<1@test>", +// "Headers": ["Subject: one", "From: a@b.example"], "Body": ["text", ".dot"]}]}} +// +// Groups take Name, Description, Created (RFC 3339, default 2020-01-01) and NoPosting; +// Articles take Group, Number, MessageID, Headers, Body (lines, no CRLF, not dot-stuffed), Date +// and Arrived (RFC 3339, default 2020-01-02T03:04:05Z). +// +// Peer mode: +// +// {"Mode": "peer", "PeerOptions": {"Have": {"": true}, "Reject": {"": true}, +// "Defer436Once": {"": true}, "User": "u", "Pass": "p", "Compress": true}} +// +// Script mode: +// +// {"Mode": "script", "Options": {"Script": [{"Expect": "CAPABILITIES", "Reply": ["101 caps", "VERSION 2", "."]}]}} +// +// Raw mode: Raw is a list of steps. Each step reads one client line that matches Expect (an +// anchored regexp; null or absent: read nothing), then a dot block when ReadDot (DotMax > 0: +// only that many lines), writes its Send chunks, waits Sleep ms and then does Then: "" (go on), +// "close", "rst" (reset the TCP connection), "hang" (keep reading, never answer), "starttls" +// (handshake on the open connection, then read through TLS) or "inflate" (read the client +// through raw DEFLATE from now on; the compressed replies are in the script). A chunk is Text +// (UTF-8) or B64 (base64), written after Gap ms, in one write or in pieces of Split bytes with +// SplitGap ms between them. A line that does not match gets "500 unexpected ..." and is a +// mismatch, as is every line after the last step. The script writes the greeting itself (a first +// step with Expect null). +// +// {"Mode": "raw", "Raw": [ +// {"Send": [{"Text": "200 raw ready\r\n"}]}, +// {"Expect": "GROUP .*", "Send": [{"Text": "211 1 1 1 alt.a\r\n", "Split": 3, "SplitGap": 1}]}, +// {"Expect": "STARTTLS", "Send": [{"Text": "382 go\r\n"}], "Then": "starttls"}, +// {"Expect": "QUIT", "Send": [{"B64": "MjA1IGJ5ZQ0K"}], "Then": "close"}]} +// +// The certificate (tlscert.GenerateSelfSigned) and the CA file are made on every start and live +// in a temporary directory that is removed on exit. +package main + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "syscall" + "time" + + "github.com/go-while/nntp-testkit/fakeserver" + "github.com/go-while/nntp-testkit/tlscert" +) + +// defaultDeadline is Options.Deadline when the config sets none. +const defaultDeadline = 10 * time.Second + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + code := run(ctx, os.Args[1:], os.Stdin, os.Stdout, os.Stderr) + stop() + os.Exit(code) +} + +// ready is the line on stdout when the server listens. +type ready struct { + Addr string `json:"addr"` + Host string `json:"host"` + Port int `json:"port"` + CA string `json:"ca"` +} + +// summary is the line on stdout when the server has stopped. +type summary struct { + Lines []string `json:"lines"` + Mismatches []string `json:"mismatches"` + Articles [][]string `json:"articles"` + Notes []string `json:"notes"` + Accepted int `json:"accepted"` + Counts map[string]int `json:"counts"` + TLSHandshakes int `json:"tls_handshakes"` + TLSFailed int `json:"tls_failed"` + ServerNames []string `json:"server_names"` +} + +// run is main without os.Exit: it serves until stdin reaches end of file or ctx is done, and +// returns the exit status. +func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("nntp-fakeserver", flag.ContinueOnError) + fs.SetOutput(stderr) + listen := fs.String("listen", "", "listen address `host:port` (default Options.Addr of the config, else 127.0.0.1:0)") + fs.Usage = func() { + fmt.Fprintf(stderr, "usage: nntp-fakeserver [flags] config.json\n\nflags:\n") + fs.PrintDefaults() + fmt.Fprintf(stderr, "\nprints {addr, host, port, ca} when listening, serves until stdin reaches end of file,\n"+ + "then prints {lines, mismatches, articles, notes, accepted, counts, tls_handshakes, tls_failed,\n"+ + "server_names} and exits; see the package documentation for the config\n") + } + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + fail := func(code int, f string, a ...any) int { + fmt.Fprintf(stderr, "nntp-fakeserver: "+f+"\n", a...) + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(stderr, "nntp-fakeserver: want exactly one argument, the config file") + fs.Usage() + return 2 + } + data, err := os.ReadFile(fs.Arg(0)) + if err != nil { + return fail(2, "%v", err) + } + + dir, err := os.MkdirTemp("", "nntp-fakeserver-") + if err != nil { + return fail(1, "%v", err) + } + defer os.RemoveAll(dir) + leaf, err := tlscert.GenerateSelfSigned(tlscert.Options{CommonName: "nntp-fakeserver"}) + if err != nil { + return fail(1, "%v", err) + } + files, err := leaf.WriteFiles(dir, 0o600) + if err != nil { + return fail(1, "%v", err) + } + tlsCfg := leaf.ServerConfig() + + cfg, err := parseConfig(data, tlsCfg) + if err != nil { + return fail(2, "%s: %v", fs.Arg(0), err) + } + var store *fakeserver.Store + if cfg.Mode == "store" { + if store, err = buildStore(cfg.Store); err != nil { + return fail(2, "%s: %v", fs.Arg(0), err) + } + } + srv, rec, err := start(cfg, store, *listen, tlsCfg) + if err != nil { + return fail(1, "%v", err) + } + defer srv.Close() + + out := json.NewEncoder(stdout) + out.SetEscapeHTML(false) + if err := out.Encode(ready{Addr: srv.Addr(), Host: srv.Host(), Port: srv.Port(), CA: files.CA}); err != nil { + return fail(1, "stdout: %v", err) + } + + eof := make(chan struct{}) + go func() { // the caller closes stdin when its client is done + _, _ = io.Copy(io.Discard, stdin) + close(eof) + }() + select { + case <-eof: + case <-ctx.Done(): + } + srv.Close() // waits for the connection goroutines: what they recorded is complete + + rec.mu.Lock() + defer rec.mu.Unlock() + s := summary{ + Lines: srv.Lines(), Mismatches: srv.Mismatches(), Articles: srv.Articles(), Notes: rec.notes, + Accepted: srv.Accepted(), Counts: srv.Counts(), + TLSHandshakes: srv.TLSHandshakes(), TLSFailed: srv.TLSFailed(), ServerNames: srv.TLSServerNames(), + } + if cfg.Mode == "raw" { + s.Lines, s.Mismatches, s.Articles, s.Counts = rec.lines, rec.mismatches, rec.articles, rec.counts + } + s.Lines, s.Mismatches, s.Notes, s.ServerNames = nz(s.Lines), nz(s.Mismatches), nz(s.Notes), nz(s.ServerNames) + if s.Articles == nil { + s.Articles = [][]string{} + } + if s.Counts == nil { + s.Counts = map[string]int{} + } + if err := out.Encode(s); err != nil { + return fail(1, "stdout: %v", err) + } + return 0 +} + +// start starts the server of cfg (store: the Store of mode store); listen, when not empty, +// overrides Options.Addr. +func start(cfg *config, store *fakeserver.Store, listen string, tlsCfg *tls.Config) (*fakeserver.Server, *recorder, error) { + o := cfg.Options + if listen != "" { + o.Addr = listen + } + if o.Deadline <= 0 { + o.Deadline = defaultDeadline + } + rec := &recorder{} + var srv *fakeserver.Server + var err error + switch cfg.Mode { + case "store": + o.Store = store + srv, err = fakeserver.Start(o) + case "peer": + po := cfg.PeerOptions + srv, err = fakeserver.StartPeer(o, &po) + case "script": + srv, err = fakeserver.Start(o) + case "raw": + o.NoGreeting = true // the script writes the greeting + o.Handler = rawHandler(cfg.Raw, rec, tlsCfg) + srv, err = fakeserver.Start(o) + } + return srv, rec, err +} + +func nz(s []string) []string { + if s == nil { + return []string{} + } + return s +} diff --git a/cmd/nntp-fakeserver/main_test.go b/cmd/nntp-fakeserver/main_test.go new file mode 100644 index 0000000..18fd93a --- /dev/null +++ b/cmd/nntp-fakeserver/main_test.go @@ -0,0 +1,735 @@ +package main + +import ( + "bufio" + "bytes" + "compress/flate" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "io" + "net" + "os" + "path/filepath" + "reflect" + "slices" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/go-while/nntp-testkit/client" + "github.com/go-while/nntp-testkit/fakeserver" +) + +// proc is one in-process run of the program, driven as a caller in another language would: the +// config in a file, the ready line on stdout, stdin closed when the client is done. +type proc struct { + t *testing.T + ready ready + stdin *io.PipeWriter + out *bufio.Reader + stderr *bytes.Buffer + code chan int + cancel context.CancelFunc +} + +func writeConfig(t *testing.T, cfg string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(p, []byte(cfg), 0o600); err != nil { + t.Fatal(err) + } + return p +} + +// startProc starts run with the config and waits for the ready line. +func startProc(t *testing.T, cfg string, flags ...string) *proc { + t.Helper() + p := launch(t, cfg, flags...) + line, err := p.out.ReadString('\n') + if err != nil { + t.Fatalf("no ready line: %v\nstderr: %s", err, p.stderr.String()) + } + if err := json.Unmarshal([]byte(line), &p.ready); err != nil { + t.Fatalf("ready line %q: %v", line, err) + } + return p +} + +// launch starts run in a goroutine; the caller reads p.out. +func launch(t *testing.T, cfg string, flags ...string) *proc { + t.Helper() + inR, inW := io.Pipe() + outR, outW := io.Pipe() + ctx, cancel := context.WithCancel(context.Background()) + p := &proc{t: t, stdin: inW, out: bufio.NewReader(outR), stderr: &bytes.Buffer{}, code: make(chan int, 1), cancel: cancel} + args := append(slices.Clone(flags), writeConfig(t, cfg)) + go func() { + code := run(ctx, args, inR, outW, p.stderr) + _ = outW.Close() + p.code <- code + }() + t.Cleanup(func() { + cancel() + _ = inW.Close() + _ = outR.Close() + }) + return p +} + +// finish closes stdin like the caller does and returns the summary and the exit status. +func (p *proc) finish() (summary, int) { + p.t.Helper() + _ = p.stdin.Close() + line, err := p.out.ReadString('\n') + if err != nil { + p.t.Fatalf("no summary line: %v\nstderr: %s", err, p.stderr.String()) + } + var s summary + dec := json.NewDecoder(strings.NewReader(line)) + dec.DisallowUnknownFields() + if err := dec.Decode(&s); err != nil { + p.t.Fatalf("summary %q: %v", line, err) + } + select { + case code := <-p.code: + return s, code + case <-time.After(10 * time.Second): + p.t.Fatal("run did not return") + return s, -1 + } +} + +func (p *proc) dial() *client.Conn { + p.t.Helper() + c, err := client.TryDial(p.ready.Addr, 10*time.Second) + if err != nil { + p.t.Fatalf("dial: %v", err) + } + p.t.Cleanup(c.Close) + return c +} + +func (p *proc) caPool() *x509.CertPool { + p.t.Helper() + pem, err := os.ReadFile(p.ready.CA) + if err != nil { + p.t.Fatal(err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + p.t.Fatalf("no certificate in %s", p.ready.CA) + } + return pool +} + +func cmd(t *testing.T, c *client.Conn, want int, format string, args ...any) string { + t.Helper() + code, line, err := c.TryCmd(format, args...) + if err != nil || code != want { + t.Fatalf("%s: got %d %q err %v, want %d", format, code, line, err, want) + } + return line +} + +const storeCfg = `{"Mode": "store", + "Options": {"Faults": {"DropInMultiline": {"ARTICLE": 2}, "Delay": {"DATE": 300}}}, + "Store": { + "Groups": [{"Name": "alt.test", "Description": "test group"}, {"Name": "alt.empty"}], + "Articles": [ + {"Group": "alt.test", "Number": 1, "MessageID": "<1@test>", + "Headers": ["Subject: one", "From: a@b.example"], "Body": ["text", ".dot"]}, + {"Group": "alt.test", "Number": 2, "MessageID": "<2@test>", + "Headers": ["Subject: two"], "Body": ["b"], "Date": "2021-03-04T05:06:07Z"}]}}` + +// TestStoreWithFaults: the Store is served, a fault applies (a drop in the second line of a +// reply, a delay in ms), and the summary says what the server saw. +func TestStoreWithFaults(t *testing.T) { + p := startProc(t, storeCfg) + if p.ready.Host != "127.0.0.1" || p.ready.Addr != net.JoinHostPort(p.ready.Host, strconv.Itoa(p.ready.Port)) || p.ready.CA == "" { + t.Errorf("ready line %+v", p.ready) + } + c := p.dial() + if line := cmd(t, c, 211, "GROUP alt.test"); line != "211 2 1 2 alt.test" { + t.Errorf("GROUP: %q", line) + } + start := time.Now() + cmd(t, c, 111, "DATE") + if d := time.Since(start); d < 250*time.Millisecond { + t.Errorf("Delay DATE 300 ms took %v", d) + } + cmd(t, c, 220, "ARTICLE 1") + lines, err := c.TryReadDotLines() + if err == nil { + t.Fatalf("ARTICLE was not cut: %q", lines) + } + s, code := p.finish() + if code != 0 { + t.Fatalf("exit %d, stderr %s", code, p.stderr.String()) + } + if want := []string{"GROUP alt.test", "DATE", "ARTICLE 1"}; !slices.Equal(s.Lines, want) { + t.Errorf("lines %q, want %q", s.Lines, want) + } + if s.Accepted != 1 || s.Counts["GROUP"] != 1 || s.Counts["ARTICLE"] != 1 { + t.Errorf("accepted %d counts %v", s.Accepted, s.Counts) + } + if len(s.Mismatches) != 0 || len(s.Notes) != 0 || len(s.Articles) != 0 || s.TLSHandshakes != 0 { + t.Errorf("summary %+v", s) + } + if _, err := os.Stat(p.ready.CA); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the CA file is still there after exit: %v", err) + } +} + +// TestStoreDates: Date, Arrived and Created are RFC 3339 strings; the defaults are fixed. +func TestStoreDates(t *testing.T) { + p := startProc(t, storeCfg) + c := p.dial() + cmd(t, c, 211, "GROUP alt.test") + cmd(t, c, 224, "OVER 1-2") + lines, err := c.TryReadDotLines() + if err != nil || len(lines) != 2 { + t.Fatalf("OVER: %q %v", lines, err) + } + if !strings.Contains(lines[1], "04 Mar 2021 05:06:07") { + t.Errorf("article 2 has its own Date: %q", lines[1]) + } + if !strings.Contains(lines[0], "02 Jan 2020 03:04:05") { + t.Errorf("article 1 has the default Date: %q", lines[0]) + } + p.finish() +} + +// TestPeerIhave: IHAVE against the peer, with its own PeerOptions, and the article is in the +// summary. +func TestPeerIhave(t *testing.T) { + p := startProc(t, `{"Mode": "peer", "PeerOptions": {"Have": {"": true}, "Defer436Once": {"": true}}}`) + c := p.dial() + cmd(t, c, 335, "IHAVE ") + if err := c.Send("Subject: s\r\n\r\n..dot\r\n.\r\n"); err != nil { + t.Fatal(err) + } + code, line, err := c.ReadStatus() + if err != nil || code != 235 { + t.Fatalf("after the article: %d %q %v", code, line, err) + } + cmd(t, c, 435, "IHAVE ") + cmd(t, c, 436, "IHAVE ") + cmd(t, c, 335, "IHAVE ") + s, _ := p.finish() + if len(s.Articles) != 1 || !slices.Equal(s.Articles[0], []string{"Subject: s", "", "..dot"}) { + t.Errorf("articles %q", s.Articles) + } + if s.Counts["IHAVE"] != 4 { + t.Errorf("counts %v", s.Counts) + } +} + +// TestPeerBadOption: a PeerOptions key that is a function or does not exist is refused. +func TestPeerBadOption(t *testing.T) { + for key, want := range map[string]string{ + `"OnArticle": {}`: "cannot be set from JSON", + `"HoldGreeting": true`: "cannot be set from JSON", + `"Have": 3`: "want an object", + } { + p := launch(t, `{"Mode": "peer", "PeerOptions": {`+key+`}}`) + if code := <-p.code; code != 2 || !strings.Contains(p.stderr.String(), want) { + t.Errorf("%s: exit %d, stderr %s", key, code, p.stderr.String()) + } + } +} + +// TestScriptMode: the kit's text script. +func TestScriptMode(t *testing.T) { + p := startProc(t, `{"Mode": "script", "Options": {"Greeting": "200 hi", + "Script": [{"Expect": "CAPABILITIES", "Delay": 20, "Reply": ["101 caps", "VERSION 2", "."]}, + {"Expect": "DATE", "Reply": ["111 20200102030405"]}]}}`) + c := p.dial() + if c.Greeting != "200 hi" { + t.Errorf("greeting %q", c.Greeting) + } + cmd(t, c, 101, "CAPABILITIES") + if _, err := c.TryReadDotLines(); err != nil { + t.Fatal(err) + } + cmd(t, c, 500, "NOPE") + cmd(t, c, 111, "DATE") + s, _ := p.finish() + if !slices.Equal(s.Mismatches, []string{"NOPE"}) { + t.Errorf("mismatches %q", s.Mismatches) + } +} + +const rawCfg = `{"Mode": "raw", "Raw": [ + {"Send": [{"Text": "200 raw ready\r\n"}]}, + {"Expect": "GROUP .*", "Send": [{"Text": "211 1 1 1 alt.a\r\n", "Split": 4, "SplitGap": 2, "Gap": 5}]}, + {"Expect": "POST", "ReadDot": false, "Send": [{"Text": "340 go\r\n"}]}, + {"Expect": "(?i)subject: .*", "ReadDot": true, "DotMax": 3, "Send": [{"B64": "MjQwIG9rDQo="}]}, + {"Expect": "QUIT", "Send": [{"Text": "205 bye\r\n"}], "Then": "close"}]}` + +// TestRawChunks: a reply in pieces arrives whole, base64 chunks are bytes, a dot block is read +// and recorded, a line that does not fit is a mismatch. +func TestRawChunks(t *testing.T) { + p := startProc(t, rawCfg) + c := p.dial() + cmd(t, c, 500, "NOPE") + cmd(t, c, 211, "GROUP alt.a") + cmd(t, c, 340, "POST") + if err := c.Send("Subject: x\r\n\r\nbody\r\n.\r\n"); err != nil { + t.Fatal(err) + } + if code, line, err := c.ReadStatus(); err != nil || code != 240 { + t.Fatalf("%d %q %v", code, line, err) + } + cmd(t, c, 205, "QUIT") + if _, err := c.ReadLine(); !errors.Is(err, io.EOF) { + t.Errorf("after the close: %v", err) + } + s, code := p.finish() + if code != 0 { + t.Fatalf("exit %d", code) + } + if want := []string{"NOPE", "GROUP alt.a", "POST", "Subject: x", "QUIT"}; !slices.Equal(s.Lines, want) { + t.Errorf("lines %q, want %q", s.Lines, want) + } + if !slices.Equal(s.Mismatches, []string{"NOPE"}) { + t.Errorf("mismatches %q", s.Mismatches) + } + if len(s.Articles) != 1 || !slices.Equal(s.Articles[0], []string{"", "body", "."}) { + t.Errorf("articles %q", s.Articles) + } + if s.Counts["GROUP"] != 1 || s.Counts["QUIT"] != 1 { + t.Errorf("counts %v", s.Counts) + } +} + +// TestRawSplitPieces: Split writes one piece per write, so the client sees several reads when +// it reads between them (a pause of 50 ms is far more than the server needs between writes). +func TestRawSplitPieces(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 abcdef\r\n", "Split": 4, "SplitGap": 50}]}]}`) + nc, err := net.DialTimeout("tcp", p.ready.Addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + var reads int + var got []byte + buf := make([]byte, 64) + _ = nc.SetDeadline(time.Now().Add(5 * time.Second)) + for !bytes.HasSuffix(got, []byte("\r\n")) { + n, err := nc.Read(buf) + if err != nil { + t.Fatalf("after %q: %v", got, err) + } + reads++ + got = append(got, buf[:n]...) + } + if string(got) != "200 abcdef\r\n" || reads < 2 { + t.Errorf("got %q in %d reads", got, reads) + } + p.finish() +} + +const rawTLSCfg = `{"Mode": "raw", "Raw": [ + {"Send": [{"Text": "200 raw ready\r\n"}]}, + {"Expect": "STARTTLS", "Send": [{"Text": "382 go\r\n"}], "Then": "starttls"}, + {"Expect": "DATE", "Send": [{"Text": "111 20200102030405\r\n"}]}]}` + +// TestRawStartTLS: a raw script upgrades with the program's certificate; the client verifies it +// against the CA file of the ready line, and the summary counts the handshake and the name. +func TestRawStartTLS(t *testing.T) { + p := startProc(t, rawTLSCfg) + c := p.dial() + tc, err := c.TryStartTLS(&tls.Config{RootCAs: p.caPool(), ServerName: "localhost"}) + if err != nil { + t.Fatal(err) + } + defer tc.Close() + if line := cmd(t, tc, 111, "DATE"); line != "111 20200102030405" { + t.Errorf("DATE over TLS: %q", line) + } + s, _ := p.finish() + if want := []string{"STARTTLS", "DATE"}; !slices.Equal(s.Lines, want) { + t.Errorf("lines %q, want %q", s.Lines, want) + } + if s.TLSHandshakes != 1 || !slices.Equal(s.ServerNames, []string{"localhost"}) || len(s.Mismatches) != 0 { + t.Errorf("tls_handshakes %d server_names %q mismatches %q notes %q", s.TLSHandshakes, s.ServerNames, s.Mismatches, s.Notes) + } +} + +// TestRawStartTLSPipelined: plaintext sent before the handshake is noted and the connection is +// closed, as the kit does in Store mode. +func TestRawStartTLSPipelined(t *testing.T) { + p := startProc(t, rawTLSCfg) + nc, err := net.DialTimeout("tcp", p.ready.Addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + _ = nc.SetDeadline(time.Now().Add(5 * time.Second)) + br := bufio.NewReader(nc) + if _, err := br.ReadString('\n'); err != nil { + t.Fatal(err) + } + // Both lines go in one write, so the second is already in the server's read buffer when + // the step for STARTTLS has answered. + if _, err := io.WriteString(nc, "STARTTLS\r\nDATE\r\n"); err != nil { + t.Fatal(err) + } + if _, err := io.Copy(io.Discard, br); err != nil && !errors.Is(err, syscall.ECONNRESET) { + t.Fatal(err) + } + s, _ := p.finish() + if len(s.Notes) != 1 || !strings.Contains(s.Notes[0], "pipelined") { + t.Errorf("notes %q", s.Notes) + } +} + +// TestRawRST: Then "rst" resets the connection. +func TestRawRST(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, + {"Expect": "QUIT", "Sleep": 20, "Then": "rst"}]}`) + c := p.dial() + if err := c.Send("QUIT\r\n"); err != nil { + t.Fatal(err) + } + _, err := c.ReadLine() + if !errors.Is(err, syscall.ECONNRESET) { + t.Errorf("read after rst: %v, want connection reset", err) + } + p.finish() +} + +// TestRawInflate: inflate reads the client through raw DEFLATE from then on (the replies stay +// plain here). +func TestRawInflate(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, + {"Expect": "COMPRESS DEFLATE", "Send": [{"Text": "206 ok\r\n"}], "Then": "inflate"}, + {"Expect": "DATE", "Send": [{"Text": "111 20200102030405\r\n"}]}]}`) + c := p.dial() + cmd(t, c, 206, "COMPRESS DEFLATE") + var buf bytes.Buffer + fw, err := flate.NewWriter(&buf, flate.DefaultCompression) + if err != nil { + t.Fatal(err) + } + fw.Write([]byte("DATE\r\n")) + if err := fw.Flush(); err != nil { + t.Fatal(err) + } + if err := c.Send(buf.String()); err != nil { + t.Fatal(err) + } + if code, line, err := c.ReadStatus(); err != nil || code != 111 { + t.Fatalf("DATE through DEFLATE: %d %q %v", code, line, err) + } + s, _ := p.finish() + if want := []string{"COMPRESS DEFLATE", "DATE"}; !slices.Equal(s.Lines, want) || len(s.Mismatches) != 0 { + t.Errorf("lines %q mismatches %q", s.Lines, s.Mismatches) + } +} + +// TestRawHang: hang reads and never answers, and does not close. +func TestRawHang(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, {"Expect": "QUIT", "Then": "hang"}]}`) + nc, err := net.DialTimeout("tcp", p.ready.Addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + br := bufio.NewReader(nc) + if _, err := br.ReadString('\n'); err != nil { + t.Fatal(err) + } + io.WriteString(nc, "QUIT\r\nMORE\r\n") + _ = nc.SetReadDeadline(time.Now().Add(300 * time.Millisecond)) + var ne net.Error + if line, err := br.ReadString('\n'); !errors.As(err, &ne) || !ne.Timeout() { + t.Errorf("a hanging server answered or closed: %q %v", line, err) + } + p.finish() +} + +// TestImplicitTLS: Options.TLS makes every connection TLS; the CA file verifies for localhost +// and for 127.0.0.1, and the handshakes and names are in the summary. +func TestImplicitTLS(t *testing.T) { + p := startProc(t, `{"Mode": "store", "Options": {"TLS": true}, "Store": {"Groups": [{"Name": "a.b"}]}}`) + pool := p.caPool() + for _, name := range []string{"localhost", "127.0.0.1"} { + nc, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", p.ready.Addr, &tls.Config{RootCAs: pool, ServerName: name}) + if err != nil { + t.Fatalf("verify %s: %v", name, err) + } + c := client.New(t, nc) + if _, err := c.ReadGreeting(); err != nil { + t.Fatal(err) + } + cmd(t, c, 211, "GROUP a.b") + c.Close() + } + // A client that does not trust the certificate fails; the server counts the failure. + if nc, err := tls.Dial("tcp", p.ready.Addr, &tls.Config{ServerName: "localhost"}); err == nil { + nc.Close() + t.Error("the system roots accepted the certificate") + } + s, _ := p.finish() + if s.TLSHandshakes != 2 || s.TLSFailed != 1 { + t.Errorf("tls_handshakes %d tls_failed %d", s.TLSHandshakes, s.TLSFailed) + } + if len(s.ServerNames) != 3 || s.ServerNames[0] != "localhost" || s.ServerNames[1] != "" { + t.Errorf("server_names %q", s.ServerNames) + } +} + +// TestStoreStartTLS: StoreOpts.StartTLS {} offers STARTTLS with the program's certificate, and +// the knobs of StartTLSOptions are read. +func TestStoreStartTLS(t *testing.T) { + p := startProc(t, `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {}}}}`) + c := p.dial() + tc, err := c.TryStartTLS(&tls.Config{RootCAs: p.caPool(), ServerName: "localhost"}) + if err != nil { + t.Fatal(err) + } + cmd(t, tc, 502, "STARTTLS") + tc.Close() + s, _ := p.finish() + if s.TLSHandshakes != 1 { + t.Errorf("tls_handshakes %d", s.TLSHandshakes) + } + + p = startProc(t, `{"Mode": "peer", "PeerOptions": {"StartTLS": {"Refuse": "580 no thanks", "HideCapability": true}}}`) + c = p.dial() + var se *client.StartTLSError + if _, err := c.TryStartTLS(&tls.Config{RootCAs: p.caPool()}); !errors.As(err, &se) || se.Code != 580 { + t.Errorf("refused STARTTLS: %v", err) + } + p.finish() +} + +// TestListenFlag: -listen picks the address; Options.Addr is the default. +func TestListenFlag(t *testing.T) { + addr := freeAddr(t) + p := startProc(t, `{"Mode": "store"}`, "-listen", addr) + if p.ready.Addr != addr { + t.Errorf("addr %q, want %q", p.ready.Addr, addr) + } + p.finish() + addr = freeAddr(t) + p = startProc(t, `{"Mode": "store", "Options": {"Addr": "`+addr+`"}}`) + if p.ready.Addr != addr { + t.Errorf("addr %q, want %q", p.ready.Addr, addr) + } + p.finish() + busy := fakeserver.New(t, fakeserver.Options{}) + p = launch(t, `{"Mode": "store"}`, "-listen", busy.Addr()) + if code := <-p.code; code != 1 || !strings.Contains(p.stderr.String(), "listen") { + t.Errorf("address in use: exit %d, stderr %s", code, p.stderr.String()) + } +} + +func freeAddr(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + return ln.Addr().String() +} + +// TestSummaryShape: with no client every key is there, empty lists are [] and not null. +func TestSummaryShape(t *testing.T) { + p := startProc(t, `{"Mode": "store"}`) + _ = p.stdin.Close() + line, err := p.out.ReadString('\n') + if err != nil { + t.Fatal(err) + } + var m map[string]json.RawMessage + if err := json.Unmarshal([]byte(line), &m); err != nil { + t.Fatal(err) + } + want := map[string]string{"lines": "[]", "mismatches": "[]", "articles": "[]", "notes": "[]", "accepted": "0", + "counts": "{}", "tls_handshakes": "0", "tls_failed": "0", "server_names": "[]"} + for k, v := range want { + if string(m[k]) != v { + t.Errorf("%s = %s, want %s", k, m[k], v) + } + } + if len(m) != len(want) { + t.Errorf("keys %v", m) + } + if code := <-p.code; code != 0 { + t.Errorf("exit %d", code) + } +} + +// TestSignal: a cancelled context ends the run like end of file. +func TestSignal(t *testing.T) { + p := startProc(t, `{"Mode": "store"}`) + p.cancel() + if _, err := p.out.ReadString('\n'); err != nil { + t.Fatal(err) + } + if code := <-p.code; code != 0 { + t.Errorf("exit %d", code) + } +} + +// TestBadConfig: a config error is exit 2 with the place in stderr, before anything listens. +func TestBadConfig(t *testing.T) { + cases := []struct{ name, cfg, want string }{ + {"unknown-top-key", `{"Mode": "store", "mode": "peer"}`, `config.mode: unknown key`}, + {"unknown-nested", `{"Mode": "store", "Options": {"Faults": {"Delayy": {}}}}`, `config.Options.Faults.Delayy: unknown key (the keys of Faults are Delay,`}, + {"typo-in-store", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "Bodyy": []}]}}`, `config.Store.Articles[0].Bodyy: unknown key`}, + {"no-mode", `{}`, `config.Mode: want one of store, peer, script, raw`}, + {"bad-mode", `{"Mode": "relay"}`, `"relay"`}, + {"not-json", `{"Mode": `, "unexpected end of JSON input"}, + {"trailing", `{"Mode": "store"} {}`, "invalid character"}, + {"duration-string", `{"Mode": "store", "Options": {"Faults": {"Delay": {"DATE": "2s"}}}}`, `config.Options.Faults.Delay["DATE"]: want a number of milliseconds`}, + {"wrong-type", `{"Mode": "store", "Options": {"Faults": {"CloseAfter": "x"}}}`, "config.Options.Faults.CloseAfter: want int"}, + {"handler", `{"Mode": "store", "Options": {"Handler": {}}}`, "config.Options.Handler: cannot be set from JSON"}, + {"store-object", `{"Mode": "store", "Options": {"Store": {}}}`, "config.Options.Store: cannot be set from JSON"}, + {"peer-in-store", `{"Mode": "store", "PeerOptions": {"User": "u"}}`, `config.PeerOptions is not used in mode "store"`}, + {"faults-in-raw", `{"Mode": "raw", "Options": {"Faults": {"CloseAfter": 1}}}`, `config.Options.Faults is not used in mode "raw"`}, + {"raw-in-store", `{"Mode": "store", "Raw": [{}]}`, `config.Raw is not used in mode "store"`}, + {"bad-regexp", `{"Mode": "raw", "Raw": [{"Expect": "(("}]}`, "config.Raw[0].Expect"}, + {"bad-script-regexp", `{"Mode": "script", "Options": {"Script": [{"Expect": "(("}]}}`, "config.Options.Script[0].Expect"}, + {"bad-then", `{"Mode": "raw", "Raw": [{"Then": "explode"}]}`, `config.Raw[0].Then`}, + {"bad-b64", `{"Mode": "raw", "Raw": [{"Send": [{"B64": "!!"}]}]}`, "config.Raw[0].Send[0]: B64"}, + {"b64-and-text", `{"Mode": "raw", "Raw": [{"Send": [{"B64": "AA==", "Text": "x"}]}]}`, "B64 and Text are both set"}, + {"bad-time", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "Date": "yesterday"}]}}`, "config.Store.Articles[0].Date"}, + {"empty-group", `{"Mode": "store", "Store": {"Groups": [{}]}}`, "Groups[0].Name is empty"}, + {"msgid-differs", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "MessageID": "", "Headers": ["Message-ID: "]}]}}`, "differs from Message-ID header"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := launch(t, tc.cfg) + if code := <-p.code; code != 2 || !strings.Contains(p.stderr.String(), tc.want) { + t.Errorf("exit %d, want 2 with %q\nstderr: %s", code, tc.want, p.stderr.String()) + } + if line, _ := p.out.ReadString('\n'); line != "" { + t.Errorf("stdout %q", line) + } + }) + } +} + +// TestUsage: arguments. +func TestUsage(t *testing.T) { + cases := []struct { + name string + args []string + code int + want string + }{ + {"no-arg", nil, 2, "exactly one argument"}, + {"two-args", []string{"a", "b"}, 2, "exactly one argument"}, + {"missing-file", []string{filepath.Join(t.TempDir(), "nope.json")}, 2, "no such file"}, + {"bad-flag", []string{"-nope"}, 2, "-nope"}, + {"help", []string{"-h"}, 0, "serves until stdin reaches end of file"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var out, errb bytes.Buffer + if code := run(context.Background(), tc.args, strings.NewReader(""), &out, &errb); code != tc.code { + t.Errorf("exit %d, want %d", code, tc.code) + } + if !strings.Contains(errb.String(), tc.want) || out.Len() != 0 { + t.Errorf("stderr %q, stdout %q, want stderr with %q", errb.String(), out.String(), tc.want) + } + }) + } +} + +// sample returns JSON that decodes into a value of type t, false for a type the decoder cannot +// set. +func sample(t reflect.Type) (string, bool) { + switch t { + case reflect.TypeFor[time.Duration](): + return "1.5", true + case reflect.TypeFor[time.Time](): + return `"2020-01-02T03:04:05Z"`, true + } + switch t.Kind() { + case reflect.Bool: + return "true", true + case reflect.String: + return `"x"`, true + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return "1", true + case reflect.Map: + v, ok := sample(t.Elem()) + return `{"K": ` + v + `}`, ok && t.Key().Kind() == reflect.String + case reflect.Slice: + v, ok := sample(t.Elem()) + return `[` + v + `]`, ok + case reflect.Pointer: + if t.Elem() == reflect.TypeFor[tls.Config]() { + return "true", true + } + return sample(t.Elem()) + case reflect.Struct: + if !hasExported(t) { + return "", false + } + return `{}`, true + } + return "", false +} + +func hasExported(t reflect.Type) bool { + for i := range t.NumField() { + if t.Field(i).IsExported() { + return true + } + } + return false +} + +// TestEveryKitFieldDecodes: every exported field of the kit's option types that is data (not a +// function or a channel) can be set from JSON, so a field added to the kit works without a +// change here; a new field of a shape the decoder cannot read fails this test. +func TestEveryKitFieldDecodes(t *testing.T) { + d := &decoder{tls: &tls.Config{}} + for _, v := range []any{fakeserver.Options{}, fakeserver.StoreOptions{}, fakeserver.PeerOptions{}, fakeserver.Faults{}, fakeserver.StartTLSOptions{}, fakeserver.Step{}} { + typ := reflect.TypeOf(v) + for i := range typ.NumField() { + f := typ.Field(i) + if !f.IsExported() { + continue + } + js, ok := sample(f.Type) + if !ok { + if k := f.Type.Kind(); k != reflect.Func && k != reflect.Chan && f.Name != "Store" { + t.Errorf("%s.%s (%s): not settable from JSON, and not a function or channel", typ.Name(), f.Name, f.Type) + } + continue + } + ptr := reflect.New(typ) + if err := d.decode(ptr.Elem(), json.RawMessage(`{"`+f.Name+`": `+js+`}`), typ.Name()); err != nil { + t.Errorf("%s.%s: %v", typ.Name(), f.Name, err) + continue + } + if f.Type.Kind() != reflect.Struct && ptr.Elem().Field(i).IsZero() { + t.Errorf("%s.%s stayed zero", typ.Name(), f.Name) + } + } + } +} + +// TestDurations: milliseconds, fractions allowed. +func TestDurations(t *testing.T) { + var f fakeserver.Faults + d := &decoder{} + if err := d.decode(reflect.ValueOf(&f).Elem(), json.RawMessage(`{"Delay": {"A": 2500, "B": 0.5}, "Drip": {"C": 1}}`), "f"); err != nil { + t.Fatal(err) + } + if f.Delay["A"] != 2500*time.Millisecond || f.Delay["B"] != 500*time.Microsecond || f.Drip["C"] != time.Millisecond { + t.Errorf("%v %v", f.Delay, f.Drip) + } +} diff --git a/cmd/nntp-fakeserver/raw.go b/cmd/nntp-fakeserver/raw.go new file mode 100644 index 0000000..93984f5 --- /dev/null +++ b/cmd/nntp-fakeserver/raw.go @@ -0,0 +1,197 @@ +package main + +import ( + "bufio" + "compress/flate" + "crypto/tls" + "fmt" + "io" + "net" + "regexp" + "strings" + "sync" + "time" + + "github.com/go-while/nntp-testkit/fakeserver" +) + +// recorder holds what raw mode saw (the fakeserver records the other modes; a raw step reads +// the socket itself, so the fakeserver sees no lines). +type recorder struct { + mu sync.Mutex + lines []string + mismatches []string + articles [][]string + notes []string + counts map[string]int +} + +func (r *recorder) line(s string) { + r.mu.Lock() + defer r.mu.Unlock() + r.lines = append(r.lines, s) + if r.counts == nil { + r.counts = make(map[string]int) + } + var cmd string + if f := strings.Fields(s); len(f) > 0 { + cmd = strings.ToUpper(f[0]) + } + r.counts[cmd]++ +} + +func (r *recorder) mismatch(s string) { + r.mu.Lock() + defer r.mu.Unlock() + r.mismatches = append(r.mismatches, s) +} + +func (r *recorder) note(format string, a ...any) { + r.mu.Lock() + defer r.mu.Unlock() + r.notes = append(r.notes, fmt.Sprintf(format, a...)) +} + +func (r *recorder) article(block []string) { + r.mu.Lock() + defer r.mu.Unlock() + r.articles = append(r.articles, block) +} + +func readLine(r *bufio.Reader) (string, error) { + s, err := r.ReadString('\n') + if err != nil { + return "", err + } + return strings.TrimRight(s, "\r\n"), nil +} + +// writeChunk writes data after the chunk's Gap, in one write or in pieces of Split bytes with +// SplitGap between them. +func writeChunk(w io.Writer, ch rawChunk) error { + data, err := ch.bytes() + if err != nil { + return err + } + time.Sleep(ch.Gap) + if ch.Split <= 0 { + _, err = w.Write(data) + return err + } + for i := 0; i < len(data); i += ch.Split { + if i > 0 { + time.Sleep(ch.SplitGap) + } + if _, err = w.Write(data[i:min(i+ch.Split, len(data))]); err != nil { + return err + } + } + return nil +} + +// rawHandler runs the steps on every connection, on Conn.Raw(). After the last step it records +// every line and answers it with a 500, so a client that sends more than the script expects +// shows up in mismatches. +func rawHandler(steps []rawStep, rec *recorder, tlsCfg *tls.Config) func(*fakeserver.Conn) { + res := make([]*regexp.Regexp, len(steps)) + for i, st := range steps { + if st.Expect != nil { + res[i] = regexp.MustCompile("^(?:" + *st.Expect + ")$") // checked by parseConfig + } + } + return func(c *fakeserver.Conn) { + conn := c.Raw() + tcp := func() *net.TCPConn { + nc := conn + if t, ok := nc.(*tls.Conn); ok { + nc = t.NetConn() + } + t, _ := nc.(*net.TCPConn) + return t + } + if t := tcp(); t != nil { + _ = t.SetNoDelay(true) + } + r := bufio.NewReader(conn) + var w io.Writer = conn + for i, st := range steps { + if res[i] != nil { + for { + line, err := readLine(r) + if err != nil { + return + } + rec.line(line) + if res[i].MatchString(line) { + break + } + rec.mismatch(line) + fmt.Fprintf(w, "500 unexpected %s\r\n", line) + } + } + if st.ReadDot { + var block []string + for { + line, err := readLine(r) + if err != nil { + rec.note("dot block read: %v after %d lines", err, len(block)) + return + } + block = append(block, line) + if line == "." || (st.DotMax > 0 && len(block) >= st.DotMax) { + break + } + } + rec.article(block) + } + for _, ch := range st.Send { + if err := writeChunk(w, ch); err != nil { + rec.note("write: %v", err) + return + } + } + time.Sleep(st.Sleep) + switch st.Then { + case "close": + return + case "rst": + if t := tcp(); t != nil { + _ = t.SetLinger(0) + } + return + case "hang": + for { + line, err := readLine(r) + if err != nil { + return + } + rec.line(line) + } + case "starttls": + if n := r.Buffered(); n > 0 { + rec.note("starttls: %d bytes of plaintext pipelined after the STARTTLS line", n) + return + } + // The handshake is bounded by the server's Deadline; the server counts it. + if err := c.StartTLS(tlsCfg); err != nil { + rec.note("starttls handshake: %v", err) + return + } + conn = c.Raw() + r = bufio.NewReader(conn) + w = conn + case "inflate": + r = bufio.NewReader(flate.NewReader(r)) + } + } + for { + line, err := readLine(r) + if err != nil { + return + } + rec.line(line) + rec.mismatch(line) + fmt.Fprintf(w, "500 unexpected %s\r\n", line) + } + } +} From a5646de7d170c8b4263e30579b7e2f2582529bee Mon Sep 17 00:00:00 2001 From: go-while Date: Thu, 12 Nov 2026 13:33:20 +0000 Subject: [PATCH 2/4] cmd/nntp-fakeserver: review fixes (binary-safe summary, bounded raw mode, config checks) Summary: lines_b64, mismatches_b64 and articles_raw (base64, lossless for bytes that are not UTF-8), max_open and closed; ready line: ca_pem. Store articles: HeadersB64 and BodyB64. Raw mode: pauses use Conn.Sleep (bounded by Deadline, ended by Close), reads of expected lines and writes have the Deadline, lines over 1 MiB, blocks over 16 MiB and 64 MiB per connection close the connection with a note. Config: Faults command keys must be upper case; empty {} and [] sections are fine in any mode; StartTLS "Config": false switches STARTTLS off; []byte is base64; check() guards a renamed kit field. A second SIGINT or SIGTERM during shutdown is no longer swallowed. Co-Authored-By: Claude Opus 5.5 --- cmd/nntp-fakeserver/config.go | 116 ++++++++++++-- cmd/nntp-fakeserver/main.go | 68 ++++++-- cmd/nntp-fakeserver/main_test.go | 257 ++++++++++++++++++++++++++++++- cmd/nntp-fakeserver/raw.go | 179 +++++++++++++++------ 4 files changed, 544 insertions(+), 76 deletions(-) diff --git a/cmd/nntp-fakeserver/config.go b/cmd/nntp-fakeserver/config.go index 0ddc465..3ae2ac1 100644 --- a/cmd/nntp-fakeserver/config.go +++ b/cmd/nntp-fakeserver/config.go @@ -19,9 +19,11 @@ import ( ) // config is the JSON document the program reads. Every key is the Go field name of the kit type -// (or of the types below), matched exactly; an unknown key is an error. A field of a kit type -// that is added later can be used from JSON at once, because the decoder walks the types -// (decodeValue), it does not mirror them. +// (or of the types below), matched exactly; an unknown key is an error. A field added later to +// Options, StoreOptions, PeerOptions, Faults or StartTLSOptions can be used from JSON at once, +// because the decoder walks those types (decodeValue) and does not mirror them. The types +// below (storeConfig, groupConfig, articleConfig, rawStep) are written by hand: a new field of +// fakeserver.Article or Group needs a line there. type config struct { Mode string // "store", "peer", "script" or "raw" @@ -51,13 +53,15 @@ type groupConfig struct { // articleConfig is one fakeserver.Article filed in Group; a zero Date and Arrived default to // 2020-01-02T03:04:05Z (so that replies are the same on every run). type articleConfig struct { - Group string - Number int64 - MessageID string - Headers []string - Body []string - Date time.Time // RFC 3339 - Arrived time.Time + Group string + Number int64 + MessageID string + Headers []string + Body []string + HeadersB64 []string // instead of Headers: each line in base64, for bytes that are not UTF-8 + BodyB64 []string // instead of Body, the same + Date time.Time // RFC 3339 + Arrived time.Time } // rawStep is one step of mode raw: read one client line (unless Expect is null or absent) that @@ -104,11 +108,6 @@ func parseConfig(data []byte, tlsCfg *tls.Config) (*config, error) { if err := cfg.check(); err != nil { return nil, err } - for _, p := range []*fakeserver.StartTLSOptions{cfg.Options.StoreOpts.StartTLS, cfg.PeerOptions.StartTLS} { - if p != nil && p.Config == nil { - p.Config = tlsCfg - } - } return &cfg, nil } @@ -130,10 +129,16 @@ func (cfg *config) check() error { {"Raw", reflect.ValueOf(&cfg.Raw).Elem(), "raw"}, } for _, p := range parts { - if !p.v.IsZero() && !strings.Contains(p.in, cfg.Mode) { + if !p.v.IsValid() { // a kit field was renamed or removed: this program needs the change + return fmt.Errorf("internal error: config.%s no longer exists in the kit", p.name) + } + if !isEmpty(p.v) && !strings.Contains(p.in, cfg.Mode) { return fmt.Errorf("config.%s is not used in mode %q (only in %s)", p.name, cfg.Mode, p.in) } } + if err := checkFaults(cfg.Options.Faults); err != nil { + return err + } for i, st := range cfg.Options.Script { if _, err := regexp.Compile("^(?:" + st.Expect + ")$"); err != nil { return fmt.Errorf("config.Options.Script[%d].Expect: %w", i, err) @@ -160,6 +165,43 @@ func (cfg *config) check() error { return nil } +// isEmpty reports whether v holds nothing a section would use: zero values, nil pointers, and +// maps and slices without elements (an empty section that a caller always writes is fine). A +// non-nil pointer is not empty, whatever it points to: "StartTLS": {} turns STARTTLS on. +func isEmpty(v reflect.Value) bool { + switch v.Kind() { + case reflect.Map, reflect.Slice: + return v.Len() == 0 + case reflect.Struct: + for i := range v.NumField() { + if !isEmpty(v.Field(i)) { + return false + } + } + return true + } + return v.IsZero() +} + +// checkFaults rejects a command key of Faults that is not upper case: the kit looks commands up +// upper-cased, so {"Delay": {"date": 100}} would never fire. OverrideLine is exempt, the kit +// normalises its keys (whole client lines, "article 2" is fine). +func checkFaults(f fakeserver.Faults) error { + v := reflect.ValueOf(f) + for i := range v.NumField() { + name := v.Type().Field(i).Name + if v.Field(i).Kind() != reflect.Map || name == "OverrideLine" || v.Type().Field(i).Type.Key().Kind() != reflect.String { + continue + } + for _, k := range v.Field(i).MapKeys() { + if key := k.String(); key != strings.ToUpper(key) { + return fmt.Errorf("config.Options.Faults.%s[%q]: command names are upper case (%q)", name, key, strings.ToUpper(key)) + } + } + } + return nil +} + // bytes returns the data of the chunk. func (ch rawChunk) bytes() ([]byte, error) { if ch.B64 != "" && ch.Text != "" { @@ -202,7 +244,15 @@ func buildStore(sc storeConfig) (st *fakeserver.Store, err error) { if a.Group == "" { return nil, fmt.Errorf("config.Store.Articles[%d].Group is empty", i) } - art := fakeserver.Article{Number: a.Number, MessageID: a.MessageID, Headers: a.Headers, Body: a.Body, Date: a.Date, Arrived: a.Arrived} + hdr, err := lines(a.Headers, a.HeadersB64) + if err != nil { + return nil, fmt.Errorf("config.Store.Articles[%d].Headers: %w", i, err) + } + body, err := lines(a.Body, a.BodyB64) + if err != nil { + return nil, fmt.Errorf("config.Store.Articles[%d].Body: %w", i, err) + } + art := fakeserver.Article{Number: a.Number, MessageID: a.MessageID, Headers: hdr, Body: body, Date: a.Date, Arrived: a.Arrived} if art.Date.IsZero() { art.Date = day(3, 4, 5, 2) } @@ -214,6 +264,25 @@ func buildStore(sc storeConfig) (st *fakeserver.Store, err error) { return st, nil } +// lines returns the text lines, or the lines of the base64 form (not both). +func lines(text, b64 []string) ([]string, error) { + if len(b64) == 0 { + return text, nil + } + if len(text) > 0 { + return nil, errors.New("the text form and the B64 form are both set") + } + out := make([]string, len(b64)) + for i, l := range b64 { + b, err := base64.StdEncoding.DecodeString(l) + if err != nil { + return nil, fmt.Errorf("B64[%d]: %w", i, err) + } + out[i] = string(b) + } + return out, nil +} + // decoder fills Go values from JSON by walking their types. // // The rules: a struct is a JSON object whose keys are exported field names (exact match; an @@ -229,6 +298,7 @@ type decoder struct { var ( durationType = reflect.TypeFor[time.Duration]() tlsConfigType = reflect.TypeFor[tls.Config]() + startTLSType = reflect.TypeFor[fakeserver.StartTLSOptions]() unmarshalerType = reflect.TypeFor[json.Unmarshaler]() ) @@ -310,6 +380,12 @@ func (d *decoder) decodeValue(dst reflect.Value, raw json.RawMessage, path strin } dst.Set(out) case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { // []byte is a base64 string, as in encoding/json + if err := json.Unmarshal(raw, dst.Addr().Interface()); err != nil { + return fmt.Errorf("want a base64 string") + } + return nil + } var arr []json.RawMessage if err := json.Unmarshal(raw, &arr); err != nil { return fmt.Errorf("want an array") @@ -348,6 +424,12 @@ func (d *decoder) decodeStruct(dst reflect.Value, raw json.RawMessage, path stri if err := json.Unmarshal(raw, &m); err != nil { return fmt.Errorf("want an object") } + if t == startTLSType { + // STARTTLS needs a config: this program's, unless "Config": false switches it off. + if _, set := m["Config"]; !set { + dst.FieldByName("Config").Set(reflect.ValueOf(d.tls)) + } + } for _, k := range sortedKeys(m) { f, ok := t.FieldByName(k) if !ok || !f.IsExported() || len(f.Index) != 1 { diff --git a/cmd/nntp-fakeserver/main.go b/cmd/nntp-fakeserver/main.go index 36f9206..b2e2eaf 100644 --- a/cmd/nntp-fakeserver/main.go +++ b/cmd/nntp-fakeserver/main.go @@ -5,15 +5,17 @@ // // Protocol with the caller: once the server listens, one line of JSON goes to stdout, // -// {"addr":"127.0.0.1:41233","host":"127.0.0.1","port":41233,"ca":"/tmp/nntp-fakeserver-1/ca.pem"} +// {"addr":"127.0.0.1:41233","host":"127.0.0.1","port":41233,"ca":"/tmp/nntp-fakeserver-1/ca.pem","ca_pem":"-----BEGIN CERTIFICATE-----\n..."} // // ca is a PEM file that holds the certificate of the server (it is self-signed, valid for -// localhost, 127.0.0.1 and ::1): a client verifies implicit TLS and STARTTLS against it. The +// localhost, 127.0.0.1 and ::1), ca_pem its text: a client verifies implicit TLS and STARTTLS +// against it. The // server then runs until stdin reaches end of file (or the process gets SIGINT or SIGTERM). It // prints one more line of JSON, the summary, removes its temporary directory and exits 0: // // {"lines":[...],"mismatches":[...],"articles":[[...]],"notes":[...],"accepted":1, -// "counts":{"GROUP":1},"tls_handshakes":0,"tls_failed":0,"server_names":[]} +// "counts":{"GROUP":1},"tls_handshakes":0,"tls_failed":0,"server_names":[], +// "lines_b64":[...],"mismatches_b64":[...],"articles_raw":[...],"max_open":1,"closed":1} // // lines are the client lines of all connections, mismatches those that did not fit (raw mode, // script mode), articles the dot blocks received (POST, IHAVE, TAKETHIS, raw ReadDot), notes @@ -22,6 +24,11 @@ // STARTTLS) and server_names the SNI of every ClientHello ("" when none). Exit status 2 and a // message on stderr for a bad config or flag, 1 for another failure. // +// A JSON string is UTF-8, so lines, mismatches and articles show any other byte (8-bit text, +// yEnc) as U+FFFD. lines_b64 and mismatches_b64 hold the same lines, articles_raw the blocks +// (the exact bytes up to the "." line, still dot-stuffed, one entry per block), all in base64 +// and exact. max_open is the most connections open at once, closed the number that ended. +// // The config is one JSON object. Its keys are the Go field names of the kit types, matched // exactly, and a key that does not exist is an error (a typo fails at once). Durations are // numbers of milliseconds (fractions allowed). Mode says what runs: @@ -37,7 +44,9 @@ // MaxConnsGreeting. A StartTLS that is {} (or has only the knobs Refuse, HideCapability, // AuthNeedsTLS, CloseAfterHandshake) offers STARTTLS with the same certificate. Functions, // channels and the Store itself cannot be set from JSON. A part that the mode does not use is -// an error. +// an error, unless it is empty ({} or []). "StartTLS": {"Config": false} switches STARTTLS off +// again. The command names that key the maps of Faults are upper case (a lower case key is an +// error: it would never fire), except OverrideLine, which takes whole client lines. // // Store mode, with a fault and STARTTLS: // @@ -53,7 +62,10 @@ // // Groups take Name, Description, Created (RFC 3339, default 2020-01-01) and NoPosting; // Articles take Group, Number, MessageID, Headers, Body (lines, no CRLF, not dot-stuffed), Date -// and Arrived (RFC 3339, default 2020-01-02T03:04:05Z). +// and Arrived (RFC 3339, default 2020-01-02T03:04:05Z). HeadersB64 and BodyB64 are Headers and +// Body with every line in base64, for bytes that are not UTF-8 (use one form of each). The +// replies of Faults (Override and so on) are UTF-8 only; raw mode with B64 chunks writes any +// bytes. // // Peer mode: // @@ -72,7 +84,10 @@ // through raw DEFLATE from now on; the compressed replies are in the script). A chunk is Text // (UTF-8) or B64 (base64), written after Gap ms, in one write or in pieces of Split bytes with // SplitGap ms between them. A line that does not match gets "500 unexpected ..." and is a -// mismatch, as is every line after the last step. The script writes the greeting itself (a first +// mismatch, as is every line after the last step. Pauses, reads of an expected line and +// writes are bounded by Options.Deadline and end when the server closes (hang and the lines +// after the script wait for the client without a limit); a line over 1 MiB, a dot block over 16 +// MiB or more than 64 MiB on one connection close the connection and add a note. The script writes the greeting itself (a first // step with Expect null). // // {"Mode": "raw", "Raw": [ @@ -88,6 +103,7 @@ package main import ( "context" "crypto/tls" + "encoding/base64" "encoding/json" "errors" "flag" @@ -107,10 +123,11 @@ const defaultDeadline = 10 * time.Second func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - code := run(ctx, os.Args[1:], os.Stdin, os.Stdout, os.Stderr) - stop() - os.Exit(code) + go func() { // after the first signal the next one kills the process, as it should while shutting down + <-ctx.Done() + stop() + }() + os.Exit(run(ctx, os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) } // ready is the line on stdout when the server listens. @@ -119,6 +136,8 @@ type ready struct { Host string `json:"host"` Port int `json:"port"` CA string `json:"ca"` + // CAPEM is the text of the CA file, for a caller that wants the certificate in memory. + CAPEM string `json:"ca_pem"` } // summary is the line on stdout when the server has stopped. @@ -132,6 +151,15 @@ type summary struct { TLSHandshakes int `json:"tls_handshakes"` TLSFailed int `json:"tls_failed"` ServerNames []string `json:"server_names"` + + // The lossless forms: JSON strings are UTF-8, so lines, mismatches and articles hold U+FFFD + // for any other byte. These hold the bytes as sent, in base64. + LinesB64 []string `json:"lines_b64"` // lines, one base64 string per line + MismatchesB64 []string `json:"mismatches_b64"` // mismatches + ArticlesRaw []string `json:"articles_raw"` // per block: the exact bytes up to the "." line, still dot-stuffed + + MaxOpen int `json:"max_open"` // the most connections open at once + Closed int `json:"closed"` // connections that have ended } // run is main without os.Exit: it serves until stdin reaches end of file or ctx is done, and @@ -200,7 +228,7 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. out := json.NewEncoder(stdout) out.SetEscapeHTML(false) - if err := out.Encode(ready{Addr: srv.Addr(), Host: srv.Host(), Port: srv.Port(), CA: files.CA}); err != nil { + if err := out.Encode(ready{Addr: srv.Addr(), Host: srv.Host(), Port: srv.Port(), CA: files.CA, CAPEM: string(leaf.CAPEM)}); err != nil { return fail(1, "stdout: %v", err) } @@ -221,9 +249,16 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. Lines: srv.Lines(), Mismatches: srv.Mismatches(), Articles: srv.Articles(), Notes: rec.notes, Accepted: srv.Accepted(), Counts: srv.Counts(), TLSHandshakes: srv.TLSHandshakes(), TLSFailed: srv.TLSFailed(), ServerNames: srv.TLSServerNames(), + MaxOpen: srv.MaxOpen(), Closed: srv.Closed(), } + raws := srv.ArticlesRaw() if cfg.Mode == "raw" { s.Lines, s.Mismatches, s.Articles, s.Counts = rec.lines, rec.mismatches, rec.articles, rec.counts + raws = rec.articleRaw + } + s.LinesB64, s.MismatchesB64, s.ArticlesRaw = b64s(s.Lines), b64s(s.Mismatches), []string{} + for _, b := range raws { + s.ArticlesRaw = append(s.ArticlesRaw, base64.StdEncoding.EncodeToString(b)) } s.Lines, s.Mismatches, s.Notes, s.ServerNames = nz(s.Lines), nz(s.Mismatches), nz(s.Notes), nz(s.ServerNames) if s.Articles == nil { @@ -262,12 +297,21 @@ func start(cfg *config, store *fakeserver.Store, listen string, tlsCfg *tls.Conf srv, err = fakeserver.Start(o) case "raw": o.NoGreeting = true // the script writes the greeting - o.Handler = rawHandler(cfg.Raw, rec, tlsCfg) + o.Handler = rawHandler(cfg.Raw, rec, tlsCfg, o.Deadline) srv, err = fakeserver.Start(o) } return srv, rec, err } +// b64s returns the strings in base64 (never nil). +func b64s(s []string) []string { + out := make([]string, len(s)) + for i, l := range s { + out[i] = base64.StdEncoding.EncodeToString([]byte(l)) + } + return out +} + func nz(s []string) []string { if s == nil { return []string{} diff --git a/cmd/nntp-fakeserver/main_test.go b/cmd/nntp-fakeserver/main_test.go index 18fd93a..0ba499a 100644 --- a/cmd/nntp-fakeserver/main_test.go +++ b/cmd/nntp-fakeserver/main_test.go @@ -7,11 +7,13 @@ import ( "context" "crypto/tls" "crypto/x509" + "encoding/base64" "encoding/json" "errors" "io" "net" "os" + "os/exec" "path/filepath" "reflect" "slices" @@ -554,7 +556,8 @@ func TestSummaryShape(t *testing.T) { t.Fatal(err) } want := map[string]string{"lines": "[]", "mismatches": "[]", "articles": "[]", "notes": "[]", "accepted": "0", - "counts": "{}", "tls_handshakes": "0", "tls_failed": "0", "server_names": "[]"} + "counts": "{}", "tls_handshakes": "0", "tls_failed": "0", "server_names": "[]", + "lines_b64": "[]", "mismatches_b64": "[]", "articles_raw": "[]", "max_open": "0", "closed": "0"} for k, v := range want { if string(m[k]) != v { t.Errorf("%s = %s, want %s", k, m[k], v) @@ -592,6 +595,10 @@ func TestBadConfig(t *testing.T) { {"trailing", `{"Mode": "store"} {}`, "invalid character"}, {"duration-string", `{"Mode": "store", "Options": {"Faults": {"Delay": {"DATE": "2s"}}}}`, `config.Options.Faults.Delay["DATE"]: want a number of milliseconds`}, {"wrong-type", `{"Mode": "store", "Options": {"Faults": {"CloseAfter": "x"}}}`, "config.Options.Faults.CloseAfter: want int"}, + {"lower-case-command", `{"Mode": "store", "Options": {"Faults": {"Delay": {"date": 2000}}}}`, `config.Options.Faults.Delay["date"]: command names are upper case ("DATE")`}, + {"lower-case-drop", `{"Mode": "store", "Options": {"Faults": {"DropInMultiline": {"Article": 1}}}}`, `DropInMultiline["Article"]`}, + {"both-body-forms", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "Body": ["x"], "BodyB64": ["eA=="]}]}}`, "Articles[0].Body: the text form and the B64 form are both set"}, + {"bad-body-b64", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "BodyB64": ["!"]}]}}`, "Articles[0].Body: B64[0]"}, {"handler", `{"Mode": "store", "Options": {"Handler": {}}}`, "config.Options.Handler: cannot be set from JSON"}, {"store-object", `{"Mode": "store", "Options": {"Store": {}}}`, "config.Options.Store: cannot be set from JSON"}, {"peer-in-store", `{"Mode": "store", "PeerOptions": {"User": "u"}}`, `config.PeerOptions is not used in mode "store"`}, @@ -660,12 +667,18 @@ func sample(t reflect.Type) (string, bool) { return "true", true case reflect.String: return `"x"`, true - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: return "1", true + case reflect.Float32, reflect.Float64: + return "1.5", true case reflect.Map: v, ok := sample(t.Elem()) return `{"K": ` + v + `}`, ok && t.Key().Kind() == reflect.String case reflect.Slice: + if t.Elem().Kind() == reflect.Uint8 { + return `"AAE="`, true // []byte is base64 + } v, ok := sample(t.Elem()) return `[` + v + `]`, ok case reflect.Pointer: @@ -733,3 +746,243 @@ func TestDurations(t *testing.T) { t.Errorf("%v %v", f.Delay, f.Drip) } } + +func b64(t *testing.T, s string) string { + t.Helper() + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// TestBinarySafe: bytes that are not UTF-8 come back exactly in lines_b64 and articles_raw (the +// plain fields hold U+FFFD), and a Store serves them from HeadersB64 and BodyB64. +func TestBinarySafe(t *testing.T) { + const art = "Subject: caf\xe9\r\n\r\n\xff\xfe\r\n..dot\r\n" + p := startProc(t, `{"Mode": "peer"}`) + c := p.dial() + cmd(t, c, 335, "IHAVE ") + if err := c.Send(art + ".\r\n"); err != nil { + t.Fatal(err) + } + if code, _, err := c.ReadStatus(); err != nil || code != 235 { + t.Fatalf("%d %v", code, err) + } + c.Send("XYZ caf\xe9\r\n") + cmd(t, c, 500, "DATE") + s, _ := p.finish() + if len(s.ArticlesRaw) != 1 || b64(t, s.ArticlesRaw[0]) != art { + t.Errorf("articles_raw %q", s.ArticlesRaw) + } + if !strings.Contains(s.Articles[0][0], "\ufffd") { + t.Errorf("the plain form is lossy by nature: %q", s.Articles[0]) + } + if len(s.LinesB64) != len(s.Lines) || b64(t, s.LinesB64[2]) != "XYZ caf\xe9" { + t.Errorf("lines_b64 %q lines %q", s.LinesB64, s.Lines) + } + + // Store: bytes in, bytes out. + p = startProc(t, `{"Mode": "store", "Store": {"Articles": [{"Group": "a.b", "Number": 1, "MessageID": "", + "HeadersB64": ["`+base64.StdEncoding.EncodeToString([]byte("Subject: caf\xe9"))+`"], + "BodyB64": ["`+base64.StdEncoding.EncodeToString([]byte("\xff\xfe"))+`"]}]}}`) + c = p.dial() + cmd(t, c, 222, "BODY ") + lines, err := c.TryReadDotLines() + if err != nil || len(lines) != 1 || lines[0] != "\xff\xfe" { + t.Errorf("BODY: %q %v", lines, err) + } + cmd(t, c, 221, "HEAD ") + lines, _ = c.TryReadDotLines() + if !slices.Contains(lines, "Subject: caf\xe9") { + t.Errorf("HEAD: %q", lines) + } + p.finish() + + // Raw mode: the dot block as the bytes were sent. + p = startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, {"Expect": "POST", "ReadDot": true, "Send": [{"Text": "240 ok\r\n"}]}]}`) + c = p.dial() + c.Send("POST\r\n\xe9\xff\r\n.\r\n") + if code, _, err := c.ReadStatus(); err != nil || code != 240 { + t.Fatalf("%d %v", code, err) + } + s, _ = p.finish() + if len(s.ArticlesRaw) != 1 || b64(t, s.ArticlesRaw[0]) != "\xe9\xff\r\n" { + t.Errorf("raw articles_raw %q", s.ArticlesRaw) + } +} + +// TestRawBoundedByDeadline: a raw step that sleeps for a minute ends with the server, and a +// client that says nothing is let go after the Deadline. +func TestRawBoundedByDeadline(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Options": {"Deadline": 600000}, "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, + {"Expect": "X", "Sleep": 60000, "Send": [{"Text": "1 a", "Gap": 60000, "Split": 1, "SplitGap": 60000}]}]}`) + c := p.dial() + c.Send("X\r\n") + time.Sleep(100 * time.Millisecond) + start := time.Now() + p.finish() + if d := time.Since(start); d > 3*time.Second { + t.Errorf("shutdown took %v while a step slept", d) + } + + p = startProc(t, `{"Mode": "raw", "Options": {"Deadline": 200}, "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, {"Expect": "X"}]}`) + c = p.dial() + if _, err := c.ReadLine(); !errors.Is(err, io.EOF) { + t.Errorf("a silent client was not let go: %v", err) + } + s, _ := p.finish() + if len(s.Notes) != 1 || !strings.Contains(s.Notes[0], "within the Deadline") { + t.Errorf("notes %q", s.Notes) + } +} + +// TestRawLineLimit: a line over 1 MiB closes the connection and is noted, not held in memory. +func TestRawLineLimit(t *testing.T) { + p := startProc(t, `{"Mode": "raw", "Raw": [{"Send": [{"Text": "200 hi\r\n"}]}, {"Expect": "X"}]}`) + c := p.dial() + go c.Send(strings.Repeat("a", maxLine+1<<16)) // the write may fail when the server closes + if _, err := c.ReadLine(); err == nil { + t.Error("the connection stayed open") + } + s, _ := p.finish() + if len(s.Notes) != 1 || !strings.Contains(s.Notes[0], "a line over") || len(s.Lines) != 0 { + t.Errorf("notes %q lines %d", s.Notes, len(s.Lines)) + } +} + +// TestEmptySections: sections that a caller writes empty are fine in any mode. +func TestEmptySections(t *testing.T) { + for _, cfg := range []string{ + `{"Mode": "raw", "Options": {"Faults": {"Delay": {}}}}`, + `{"Mode": "store", "Raw": [], "PeerOptions": {"Have": {}}, "Options": {"Script": []}}`, + `{"Mode": "peer", "Store": {"Groups": [], "Articles": []}, "Options": {"StoreOpts": {"OverviewFmt": []}}}`, + `{"Mode": "store", "Options": {"Faults": {"OverrideLine": {"article 2": ["430 no"]}}}}`, // the kit normalises + } { + if _, err := parseConfig([]byte(cfg), &tls.Config{}); err != nil { + t.Errorf("%s: %v", cfg, err) + } + } + for _, cfg := range []string{ + `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {}}}, "Raw": [{}]}`, + `{"Mode": "raw", "Options": {"StoreOpts": {"StartTLS": {}}}}`, // a StartTLS object is not empty + } { + if _, err := parseConfig([]byte(cfg), &tls.Config{}); err == nil { + t.Errorf("%s: accepted", cfg) + } + } +} + +// TestStartTLSConfigOff: StartTLS {} offers STARTTLS, "Config": false does not, and the +// certificate of the program is the default only when Config is not named. +func TestStartTLSConfigOff(t *testing.T) { + for cfg, want := range map[string]int{ + `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {}}}}`: 382, + `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {"Config": true}}}}`: 382, + `{"Mode": "store", "Options": {"StoreOpts": {"StartTLS": {"Config": false}}}}`: 500, + `{"Mode": "peer", "PeerOptions": {"StartTLS": {"Config": false, "Refuse": "502"}}}`: 500, + } { + p := startProc(t, cfg) + c := p.dial() + if code, line, err := c.TryCmd("STARTTLS"); err != nil || code != want { + t.Errorf("%s: STARTTLS %d %q %v, want %d", cfg, code, line, err, want) + } + p.finish() + } +} + +// TestCounters: max_open and closed, and ca_pem in the ready line. +func TestCounters(t *testing.T) { + p := startProc(t, `{"Mode": "store"}`) + pem, err := os.ReadFile(p.ready.CA) + if err != nil || p.ready.CAPEM == "" || p.ready.CAPEM != string(pem) { + t.Errorf("ca_pem %q, file %q, %v", p.ready.CAPEM, pem, err) + } + a, b := p.dial(), p.dial() + cmd(t, a, 111, "DATE") + cmd(t, b, 111, "DATE") + a.Close() + b.Close() + s, _ := p.finish() + if s.MaxOpen != 2 || s.Closed != 2 || s.Accepted != 2 { + t.Errorf("max_open %d closed %d accepted %d", s.MaxOpen, s.Closed, s.Accepted) + } +} + +// TestIsEmpty: what counts as an empty section. +func TestIsEmpty(t *testing.T) { + for _, tc := range []struct { + v any + want bool + }{ + {fakeserver.Faults{}, true}, + {fakeserver.Faults{Delay: map[string]time.Duration{}}, true}, + {fakeserver.Faults{Delay: map[string]time.Duration{"A": 0}}, false}, + {fakeserver.StoreOptions{StartTLS: &fakeserver.StartTLSOptions{}}, false}, + {[]rawStep{}, true}, + {[]rawStep{{}}, false}, + } { + if got := isEmpty(reflect.ValueOf(tc.v)); got != tc.want { + t.Errorf("isEmpty(%#v) = %v", tc.v, got) + } + } +} + +// TestBytesAreBase64: a []byte field takes a base64 string, as in encoding/json. +func TestBytesAreBase64(t *testing.T) { + var v struct{ B []byte } + d := &decoder{} + if err := d.decode(reflect.ValueOf(&v).Elem(), json.RawMessage(`{"B": "AAEC"}`), "v"); err != nil || !bytes.Equal(v.B, []byte{0, 1, 2}) { + t.Errorf("%v %v", v.B, err) + } + if err := d.decode(reflect.ValueOf(&v).Elem(), json.RawMessage(`{"B": [0, 1]}`), "v"); err == nil { + t.Error("an array was taken for bytes") + } +} + +// TestMain lets TestSignal run the real main in a child process. +func TestMain(m *testing.M) { + if cfg := os.Getenv("NNTP_FAKESERVER_TEST_CONFIG"); cfg != "" { + os.Args = []string{"nntp-fakeserver", cfg} + main() + return + } + os.Exit(m.Run()) +} + +// TestMainSignal: the real main, as a process: SIGTERM ends the server with its summary and exit +// status 0, and so does EOF on stdin. +func TestMainSignal(t *testing.T) { + for _, how := range []string{"SIGTERM", "SIGINT", "EOF"} { + t.Run(how, func(t *testing.T) { + cmd := exec.Command(os.Args[0]) + cmd.Env = append(os.Environ(), "NNTP_FAKESERVER_TEST_CONFIG="+writeConfig(t, `{"Mode": "store", "Options": {"RawStall": true}}`)) + in, _ := cmd.StdinPipe() + out, _ := cmd.StdoutPipe() + var errb bytes.Buffer + cmd.Stderr = &errb + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + br := bufio.NewReader(out) + if _, err := br.ReadString('\n'); err != nil { + t.Fatalf("%v: %s", err, errb.String()) + } + switch how { + case "SIGTERM": + cmd.Process.Signal(syscall.SIGTERM) + case "SIGINT": + cmd.Process.Signal(syscall.SIGINT) + default: + in.Close() + } + line, err := br.ReadString('\n') + if err != nil || !strings.Contains(line, `"accepted":0`) { + t.Errorf("summary %q %v", line, err) + } + if err := cmd.Wait(); err != nil { + t.Errorf("exit: %v %s", err, errb.String()) + } + }) + } +} diff --git a/cmd/nntp-fakeserver/raw.go b/cmd/nntp-fakeserver/raw.go index 93984f5..1550cf8 100644 --- a/cmd/nntp-fakeserver/raw.go +++ b/cmd/nntp-fakeserver/raw.go @@ -4,9 +4,10 @@ import ( "bufio" "compress/flate" "crypto/tls" + "errors" "fmt" - "io" "net" + "os" "regexp" "strings" "sync" @@ -22,6 +23,7 @@ type recorder struct { lines []string mismatches []string articles [][]string + articleRaw [][]byte // the blocks as read, without the "." line (articles_raw) notes []string counts map[string]int } @@ -52,37 +54,121 @@ func (r *recorder) note(format string, a ...any) { r.notes = append(r.notes, fmt.Sprintf(format, a...)) } -func (r *recorder) article(block []string) { +func (r *recorder) article(block []string, wire []byte) { r.mu.Lock() defer r.mu.Unlock() r.articles = append(r.articles, block) + r.articleRaw = append(r.articleRaw, wire) } -func readLine(r *bufio.Reader) (string, error) { - s, err := r.ReadString('\n') +// The limits of what a raw connection reads (a client or a DEFLATE bomb cannot fill the memory): +// one line, one dot block, and everything read on one connection. A connection that goes over +// one is closed and a note says so. +const ( + maxLine = 1 << 20 + maxBlock = 16 << 20 + maxConn = 64 << 20 +) + +var errTooLong = errors.New("line longer than the limit") + +// rawConn is one connection of mode raw. +type rawConn struct { + c *fakeserver.Conn + rec *recorder + deadline time.Duration // Options.Deadline: bounds every wait, read and write + conn net.Conn + r *bufio.Reader + total int // bytes read so far +} + +func (rc *rawConn) tcp() *net.TCPConn { + nc := rc.conn + if t, ok := nc.(*tls.Conn); ok { + nc = t.NetConn() + } + t, _ := nc.(*net.TCPConn) + return t +} + +// readWire reads one line with its line end. A timed read waits at most the Deadline for it +// (the read of a line the script expects); an untimed one waits for the client (hang, the lines +// after the script) and ends with the connection. A line over maxLine or a connection over +// maxConn ends the read with a note. +func (rc *rawConn) readWire(timed bool) ([]byte, error) { + if timed { + _ = rc.conn.SetReadDeadline(time.Now().Add(rc.deadline)) + } else { + _ = rc.conn.SetReadDeadline(time.Time{}) + } + var line []byte + for { + part, err := rc.r.ReadSlice('\n') + line = append(line, part...) + rc.total += len(part) + switch { + case err == nil: + if rc.total > maxConn { + rc.rec.note("read: over %d bytes on one connection, closed", maxConn) + return line, errTooLong + } + return line, nil + case errors.Is(err, bufio.ErrBufferFull): + case errors.Is(err, os.ErrDeadlineExceeded): + rc.rec.note("read: nothing from the client within the Deadline (%v)", rc.deadline) + return line, err + default: + var ce flate.CorruptInputError + if errors.As(err, &ce) { + rc.rec.note("read: %v", err) + } + return line, err + } + if len(line) > maxLine { + rc.rec.note("read: a line over %d bytes, connection closed", maxLine) + return line, errTooLong + } + if rc.total > maxConn { + rc.rec.note("read: over %d bytes on one connection, closed", maxConn) + return line, errTooLong + } + } +} + +// readLine is readWire without the line end. +func (rc *rawConn) readLine(timed bool) (string, error) { + w, err := rc.readWire(timed) if err != nil { return "", err } - return strings.TrimRight(s, "\r\n"), nil + return strings.TrimRight(string(w), "\r\n"), nil +} + +// write writes data within the Deadline. +func (rc *rawConn) write(data []byte) error { + _ = rc.conn.SetWriteDeadline(time.Now().Add(rc.deadline)) + _, err := rc.conn.Write(data) + return err } -// writeChunk writes data after the chunk's Gap, in one write or in pieces of Split bytes with -// SplitGap between them. -func writeChunk(w io.Writer, ch rawChunk) error { +// writeChunk writes the chunk after its Gap, in one write or in pieces of Split bytes with +// SplitGap between them. The pauses end when the server closes or at the Deadline. +func (rc *rawConn) writeChunk(ch rawChunk) error { data, err := ch.bytes() if err != nil { return err } - time.Sleep(ch.Gap) + if ch.Gap > 0 && !rc.c.Sleep(ch.Gap) { + return errors.New("server closed") + } if ch.Split <= 0 { - _, err = w.Write(data) - return err + return rc.write(data) } for i := 0; i < len(data); i += ch.Split { - if i > 0 { - time.Sleep(ch.SplitGap) + if i > 0 && ch.SplitGap > 0 && !rc.c.Sleep(ch.SplitGap) { + return errors.New("server closed") } - if _, err = w.Write(data[i:min(i+ch.Split, len(data))]); err != nil { + if err := rc.write(data[i:min(i+ch.Split, len(data))]); err != nil { return err } } @@ -92,7 +178,7 @@ func writeChunk(w io.Writer, ch rawChunk) error { // rawHandler runs the steps on every connection, on Conn.Raw(). After the last step it records // every line and answers it with a 500, so a client that sends more than the script expects // shows up in mismatches. -func rawHandler(steps []rawStep, rec *recorder, tlsCfg *tls.Config) func(*fakeserver.Conn) { +func rawHandler(steps []rawStep, rec *recorder, tlsCfg *tls.Config, deadline time.Duration) func(*fakeserver.Conn) { res := make([]*regexp.Regexp, len(steps)) for i, st := range steps { if st.Expect != nil { @@ -100,24 +186,15 @@ func rawHandler(steps []rawStep, rec *recorder, tlsCfg *tls.Config) func(*fakese } } return func(c *fakeserver.Conn) { - conn := c.Raw() - tcp := func() *net.TCPConn { - nc := conn - if t, ok := nc.(*tls.Conn); ok { - nc = t.NetConn() - } - t, _ := nc.(*net.TCPConn) - return t - } - if t := tcp(); t != nil { + rc := &rawConn{c: c, rec: rec, deadline: deadline, conn: c.Raw()} + if t := rc.tcp(); t != nil { _ = t.SetNoDelay(true) } - r := bufio.NewReader(conn) - var w io.Writer = conn + rc.r = bufio.NewReader(rc.conn) for i, st := range steps { if res[i] != nil { for { - line, err := readLine(r) + line, err := rc.readLine(true) if err != nil { return } @@ -126,49 +203,62 @@ func rawHandler(steps []rawStep, rec *recorder, tlsCfg *tls.Config) func(*fakese break } rec.mismatch(line) - fmt.Fprintf(w, "500 unexpected %s\r\n", line) + _ = rc.write([]byte("500 unexpected " + line + "\r\n")) } } if st.ReadDot { var block []string + var wire []byte for { - line, err := readLine(r) + w, err := rc.readWire(true) if err != nil { - rec.note("dot block read: %v after %d lines", err, len(block)) + if !errors.Is(err, errTooLong) && !errors.Is(err, os.ErrDeadlineExceeded) { + rec.note("dot block read: %v after %d lines", err, len(block)) + } return } + line := strings.TrimRight(string(w), "\r\n") block = append(block, line) + if line != "." { + wire = append(wire, w...) + } + if len(wire) > maxBlock { + rec.note("dot block: over %d bytes, connection closed", maxBlock) + return + } if line == "." || (st.DotMax > 0 && len(block) >= st.DotMax) { break } } - rec.article(block) + rec.article(block, wire) } for _, ch := range st.Send { - if err := writeChunk(w, ch); err != nil { + if err := rc.writeChunk(ch); err != nil { rec.note("write: %v", err) return } } - time.Sleep(st.Sleep) + if st.Sleep > 0 && !c.Sleep(st.Sleep) { + return + } switch st.Then { case "close": return case "rst": - if t := tcp(); t != nil { + if t := rc.tcp(); t != nil { _ = t.SetLinger(0) } return case "hang": - for { - line, err := readLine(r) + for { // holds the connection: no read deadline + line, err := rc.readLine(false) if err != nil { return } rec.line(line) } case "starttls": - if n := r.Buffered(); n > 0 { + if n := rc.r.Buffered(); n > 0 { rec.note("starttls: %d bytes of plaintext pipelined after the STARTTLS line", n) return } @@ -177,21 +267,20 @@ func rawHandler(steps []rawStep, rec *recorder, tlsCfg *tls.Config) func(*fakese rec.note("starttls handshake: %v", err) return } - conn = c.Raw() - r = bufio.NewReader(conn) - w = conn + rc.conn = c.Raw() + rc.r = bufio.NewReader(rc.conn) case "inflate": - r = bufio.NewReader(flate.NewReader(r)) + rc.r = bufio.NewReader(flate.NewReader(rc.r)) } } - for { - line, err := readLine(r) + for { // the lines after the script: no read deadline either + line, err := rc.readLine(false) if err != nil { return } rec.line(line) rec.mismatch(line) - fmt.Fprintf(w, "500 unexpected %s\r\n", line) + _ = rc.write([]byte("500 unexpected " + line + "\r\n")) } } } From 8f8c47625b677677e82fc6fddffd657719cdbf29 Mon Sep 17 00:00:00 2001 From: go-while Date: Thu, 12 Nov 2026 13:39:20 +0000 Subject: [PATCH 3/4] cmd/nntp-fakeserver: test the byte-level Faults of the merged kit, []byte must be a string TestEveryKitFieldDecodes now builds nested samples (Split, Trailing); TestByteFaults checks their values and a Split + Trailing reply end to end. A []byte field takes a base64 string only (encoding/json would also take an array). Co-Authored-By: Claude Opus 5.5 --- cmd/nntp-fakeserver/config.go | 2 +- cmd/nntp-fakeserver/main_test.go | 61 ++++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/cmd/nntp-fakeserver/config.go b/cmd/nntp-fakeserver/config.go index 3ae2ac1..b0b3097 100644 --- a/cmd/nntp-fakeserver/config.go +++ b/cmd/nntp-fakeserver/config.go @@ -381,7 +381,7 @@ func (d *decoder) decodeValue(dst reflect.Value, raw json.RawMessage, path strin dst.Set(out) case reflect.Slice: if t.Elem().Kind() == reflect.Uint8 { // []byte is a base64 string, as in encoding/json - if err := json.Unmarshal(raw, dst.Addr().Interface()); err != nil { + if !bytes.HasPrefix(bytes.TrimSpace(raw), []byte(`"`)) || json.Unmarshal(raw, dst.Addr().Interface()) != nil { return fmt.Errorf("want a base64 string") } return nil diff --git a/cmd/nntp-fakeserver/main_test.go b/cmd/nntp-fakeserver/main_test.go index 0ba499a..0c89666 100644 --- a/cmd/nntp-fakeserver/main_test.go +++ b/cmd/nntp-fakeserver/main_test.go @@ -690,7 +690,15 @@ func sample(t reflect.Type) (string, bool) { if !hasExported(t) { return "", false } - return `{}`, true + var parts []string // every field that can be set, so a nested struct is exercised too + for i := range t.NumField() { + if f := t.Field(i); f.IsExported() { + if v, ok := sample(f.Type); ok { + parts = append(parts, `"`+f.Name+`": `+v) + } + } + } + return `{` + strings.Join(parts, ", ") + `}`, true } return "", false } @@ -778,7 +786,7 @@ func TestBinarySafe(t *testing.T) { if !strings.Contains(s.Articles[0][0], "\ufffd") { t.Errorf("the plain form is lossy by nature: %q", s.Articles[0]) } - if len(s.LinesB64) != len(s.Lines) || b64(t, s.LinesB64[2]) != "XYZ caf\xe9" { + if len(s.LinesB64) != len(s.Lines) || b64(t, s.LinesB64[1]) != "XYZ caf\xe9" { t.Errorf("lines_b64 %q lines %q", s.LinesB64, s.Lines) } @@ -986,3 +994,52 @@ func TestMainSignal(t *testing.T) { }) } } + +// TestByteFaults: the byte-level Faults (Split, Trailing, DropAfterBytes, Abort, OverrideFinal, +// DelayFinal, DropInUpload) come through with their nested fields and durations in ms. +func TestByteFaults(t *testing.T) { + cfg, err := parseConfig([]byte(`{"Mode": "store", "Options": {"Faults": { + "Split": {"DATE": {"At": [1, 5], "Size": 3, "Gap": 2.5}}, + "Trailing": {"DATE": {"Lines": ["111 x", "."], "Gap": 40}}, + "DropAfterBytes": {"GROUP": 4}, "Abort": {"GROUP": true}, + "OverrideFinal": {"POST": ["441 no"]}, "DelayFinal": {"POST": 15}, "DropInUpload": {"POST": 2}}}}`), &tls.Config{}) + if err != nil { + t.Fatal(err) + } + f := cfg.Options.Faults + if sp := f.Split["DATE"]; !slices.Equal(sp.At, []int{1, 5}) || sp.Size != 3 || sp.Gap != 2500*time.Microsecond { + t.Errorf("Split %+v", sp) + } + if tr := f.Trailing["DATE"]; !slices.Equal(tr.Lines, []string{"111 x", "."}) || tr.Gap != 40*time.Millisecond { + t.Errorf("Trailing %+v", tr) + } + if f.DropAfterBytes["GROUP"] != 4 || !f.Abort["GROUP"] || f.OverrideFinal["POST"][0] != "441 no" || + f.DelayFinal["POST"] != 15*time.Millisecond || f.DropInUpload["POST"] != 2 { + t.Errorf("faults %+v", f) + } + + // End to end: the status line of DATE in two writes with a pause, then the trailing line. + p := startProc(t, `{"Mode": "store", "Options": {"Faults": { + "Split": {"DATE": {"At": [3], "Gap": 100}}, "Trailing": {"DATE": {"Lines": ["TRAILER"], "Gap": 50}}}}}`) + nc, err := net.DialTimeout("tcp", p.ready.Addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + br := bufio.NewReader(nc) + br.ReadString('\n') + io.WriteString(nc, "DATE\r\n") + _ = nc.SetReadDeadline(time.Now().Add(5 * time.Second)) + var got []string + for range 2 { + l, err := br.ReadString('\n') + if err != nil { + t.Fatal(err) + } + got = append(got, strings.TrimSpace(l)) + } + if !strings.HasPrefix(got[0], "111 ") || got[1] != "TRAILER" { + t.Errorf("got %q", got) + } + p.finish() +} From 644dfd03bdabadf2c83ef85f5acb9f8ba0984b01 Mon Sep 17 00:00:00 2001 From: go-while Date: Thu, 12 Nov 2026 14:03:41 +0000 Subject: [PATCH 4/4] cmd/nntp-fakeserver: re-review fixes (pauses over Deadline, B64 key names, usage keys) - A raw Sleep, Gap or SplitGap longer than Options.Deadline is a config error: Conn.Sleep ends every pause at the Deadline, so it was cut short without a note. The package doc says a client that stays silent longer than the Deadline before an expected line is cut off. - Base64 errors name their key (HeadersB64[0], BodyB64[0]), not Headers or Body. - -h lists the current ready and summary keys. Co-Authored-By: Claude Opus 5.5 --- cmd/nntp-fakeserver/config.go | 36 +++++++++++++++++++++++++------- cmd/nntp-fakeserver/main.go | 16 ++++++++------ cmd/nntp-fakeserver/main_test.go | 10 +++++++-- 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/cmd/nntp-fakeserver/config.go b/cmd/nntp-fakeserver/config.go index b0b3097..91139a7 100644 --- a/cmd/nntp-fakeserver/config.go +++ b/cmd/nntp-fakeserver/config.go @@ -147,7 +147,29 @@ func (cfg *config) check() error { if cfg.Mode != "raw" { return nil } + deadline := cfg.Options.Deadline + if deadline <= 0 { + deadline = defaultDeadline + } + // Conn.Sleep ends every pause at the Deadline, so a longer one would be cut short silently + tooLong := func(field string, d time.Duration) error { + if d > deadline { + return fmt.Errorf("config.%s: %v is longer than Options.Deadline (%v), which ends every pause; raise Options.Deadline", field, d, deadline) + } + return nil + } for i, st := range cfg.Raw { + if err := tooLong(fmt.Sprintf("Raw[%d].Sleep", i), st.Sleep); err != nil { + return err + } + for j, ch := range st.Send { + if err := errors.Join( + tooLong(fmt.Sprintf("Raw[%d].Send[%d].Gap", i, j), ch.Gap), + tooLong(fmt.Sprintf("Raw[%d].Send[%d].SplitGap", i, j), ch.SplitGap), + ); err != nil { + return err + } + } if st.Expect != nil { if _, err := regexp.Compile("^(?:" + *st.Expect + ")$"); err != nil { return fmt.Errorf("config.Raw[%d].Expect: %w", i, err) @@ -244,13 +266,13 @@ func buildStore(sc storeConfig) (st *fakeserver.Store, err error) { if a.Group == "" { return nil, fmt.Errorf("config.Store.Articles[%d].Group is empty", i) } - hdr, err := lines(a.Headers, a.HeadersB64) + hdr, err := lines("Headers", a.Headers, a.HeadersB64) if err != nil { - return nil, fmt.Errorf("config.Store.Articles[%d].Headers: %w", i, err) + return nil, fmt.Errorf("config.Store.Articles[%d].%w", i, err) } - body, err := lines(a.Body, a.BodyB64) + body, err := lines("Body", a.Body, a.BodyB64) if err != nil { - return nil, fmt.Errorf("config.Store.Articles[%d].Body: %w", i, err) + return nil, fmt.Errorf("config.Store.Articles[%d].%w", i, err) } art := fakeserver.Article{Number: a.Number, MessageID: a.MessageID, Headers: hdr, Body: body, Date: a.Date, Arrived: a.Arrived} if art.Date.IsZero() { @@ -265,18 +287,18 @@ func buildStore(sc storeConfig) (st *fakeserver.Store, err error) { } // lines returns the text lines, or the lines of the base64 form (not both). -func lines(text, b64 []string) ([]string, error) { +func lines(name string, text, b64 []string) ([]string, error) { if len(b64) == 0 { return text, nil } if len(text) > 0 { - return nil, errors.New("the text form and the B64 form are both set") + return nil, fmt.Errorf("%s and %sB64 are both set", name, name) } out := make([]string, len(b64)) for i, l := range b64 { b, err := base64.StdEncoding.DecodeString(l) if err != nil { - return nil, fmt.Errorf("B64[%d]: %w", i, err) + return nil, fmt.Errorf("%sB64[%d]: %w", name, i, err) } out[i] = string(b) } diff --git a/cmd/nntp-fakeserver/main.go b/cmd/nntp-fakeserver/main.go index b2e2eaf..091fe00 100644 --- a/cmd/nntp-fakeserver/main.go +++ b/cmd/nntp-fakeserver/main.go @@ -86,9 +86,11 @@ // SplitGap ms between them. A line that does not match gets "500 unexpected ..." and is a // mismatch, as is every line after the last step. Pauses, reads of an expected line and // writes are bounded by Options.Deadline and end when the server closes (hang and the lines -// after the script wait for the client without a limit); a line over 1 MiB, a dot block over 16 -// MiB or more than 64 MiB on one connection close the connection and add a note. The script writes the greeting itself (a first -// step with Expect null). +// after the script wait for the client without a limit): a client that stays silent longer than +// the Deadline before an expected line is cut off, and a Sleep, Gap or SplitGap longer than the +// Deadline is a config error. A line over 1 MiB, a dot block over 16 MiB or more than 64 MiB on +// one connection close the connection and add a note. The script writes the greeting itself (a +// first step with Expect null). // // {"Mode": "raw", "Raw": [ // {"Send": [{"Text": "200 raw ready\r\n"}]}, @@ -171,9 +173,11 @@ func run(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io. fs.Usage = func() { fmt.Fprintf(stderr, "usage: nntp-fakeserver [flags] config.json\n\nflags:\n") fs.PrintDefaults() - fmt.Fprintf(stderr, "\nprints {addr, host, port, ca} when listening, serves until stdin reaches end of file,\n"+ - "then prints {lines, mismatches, articles, notes, accepted, counts, tls_handshakes, tls_failed,\n"+ - "server_names} and exits; see the package documentation for the config\n") + fmt.Fprintf(stderr, "\nprints {addr, host, port, ca, ca_pem} when listening,\n"+ + "serves until stdin reaches end of file,\n"+ + "then prints {lines, lines_b64, mismatches, mismatches_b64, articles, articles_raw, notes,\n"+ + "accepted, counts, tls_handshakes, tls_failed, server_names, max_open, closed} and exits;\n"+ + "see the package documentation for the config\n") } if err := fs.Parse(args); err != nil { if errors.Is(err, flag.ErrHelp) { diff --git a/cmd/nntp-fakeserver/main_test.go b/cmd/nntp-fakeserver/main_test.go index 0c89666..490869b 100644 --- a/cmd/nntp-fakeserver/main_test.go +++ b/cmd/nntp-fakeserver/main_test.go @@ -597,8 +597,12 @@ func TestBadConfig(t *testing.T) { {"wrong-type", `{"Mode": "store", "Options": {"Faults": {"CloseAfter": "x"}}}`, "config.Options.Faults.CloseAfter: want int"}, {"lower-case-command", `{"Mode": "store", "Options": {"Faults": {"Delay": {"date": 2000}}}}`, `config.Options.Faults.Delay["date"]: command names are upper case ("DATE")`}, {"lower-case-drop", `{"Mode": "store", "Options": {"Faults": {"DropInMultiline": {"Article": 1}}}}`, `DropInMultiline["Article"]`}, - {"both-body-forms", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "Body": ["x"], "BodyB64": ["eA=="]}]}}`, "Articles[0].Body: the text form and the B64 form are both set"}, - {"bad-body-b64", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "BodyB64": ["!"]}]}}`, "Articles[0].Body: B64[0]"}, + {"both-body-forms", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "Body": ["x"], "BodyB64": ["eA=="]}]}}`, "Articles[0].Body and BodyB64 are both set"}, + {"bad-body-b64", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "BodyB64": ["!"]}]}}`, "Articles[0].BodyB64[0]:"}, + {"bad-headers-b64", `{"Mode": "store", "Store": {"Articles": [{"Group": "a", "HeadersB64": ["!!"]}]}}`, "config.Store.Articles[0].HeadersB64[0]:"}, + {"sleep-over-deadline", `{"Mode": "raw", "Raw": [{"Sleep": 30000}]}`, "config.Raw[0].Sleep: 30s is longer than Options.Deadline (10s)"}, + {"gap-over-deadline", `{"Mode": "raw", "Options": {"Deadline": 500}, "Raw": [{"Send": [{"Text": "x", "Gap": 600}]}]}`, "config.Raw[0].Send[0].Gap: 600ms is longer than Options.Deadline (500ms)"}, + {"splitgap-over-deadline", `{"Mode": "raw", "Raw": [{"Send": [{"Text": "xy", "Split": 1, "SplitGap": 10001}]}]}`, "config.Raw[0].Send[0].SplitGap"}, {"handler", `{"Mode": "store", "Options": {"Handler": {}}}`, "config.Options.Handler: cannot be set from JSON"}, {"store-object", `{"Mode": "store", "Options": {"Store": {}}}`, "config.Options.Store: cannot be set from JSON"}, {"peer-in-store", `{"Mode": "store", "PeerOptions": {"User": "u"}}`, `config.PeerOptions is not used in mode "store"`}, @@ -639,6 +643,8 @@ func TestUsage(t *testing.T) { {"missing-file", []string{filepath.Join(t.TempDir(), "nope.json")}, 2, "no such file"}, {"bad-flag", []string{"-nope"}, 2, "-nope"}, {"help", []string{"-h"}, 0, "serves until stdin reaches end of file"}, + {"help-ready-keys", []string{"-h"}, 0, "{addr, host, port, ca, ca_pem}"}, + {"help-summary-keys", []string{"-h"}, 0, "server_names, max_open, closed}"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) {