From 133b07ab73b820a8be66818d05dffa48cab2c6a5 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Fri, 26 Jan 2024 16:33:58 -0600 Subject: [PATCH] fixturescript: a tiny scripting engine for specifying test fixtures Signed-off-by: Hank Donnay --- go.mod | 1 + go.sum | 2 + test/fixturescript/fixturescript.go | 171 +++++++++++++++++++++++++ test/fixturescript/indexreport.go | 156 ++++++++++++++++++++++ test/fixturescript/indexreport_test.go | 49 +++++++ 5 files changed, 379 insertions(+) create mode 100644 test/fixturescript/fixturescript.go create mode 100644 test/fixturescript/indexreport.go create mode 100644 test/fixturescript/indexreport_test.go diff --git a/go.mod b/go.mod index eaeff3d60..afc2d212d 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.5.0 + github.com/hugelgupf/go-shlex v0.0.0-20200702092117-c80c9d0918fa github.com/jackc/pgconn v1.14.1 github.com/jackc/pgtype v1.14.0 github.com/jackc/pgx/v4 v4.18.0 diff --git a/go.sum b/go.sum index 8b1035b45..7dcb945c3 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbu github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hugelgupf/go-shlex v0.0.0-20200702092117-c80c9d0918fa h1:s3KPo0nThtvjEamF/aElD4k5jSsBHew3/sgNTnth+2M= +github.com/hugelgupf/go-shlex v0.0.0-20200702092117-c80c9d0918fa/go.mod h1:I1uW6ymzwsy5TlQgD1bFAghdMgBYqH1qtCeHoZgHMqs= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= diff --git a/test/fixturescript/fixturescript.go b/test/fixturescript/fixturescript.go new file mode 100644 index 000000000..6cc29b432 --- /dev/null +++ b/test/fixturescript/fixturescript.go @@ -0,0 +1,171 @@ +// Package fixturescript is a small language to declare claircore test fixtures. +// +// Fixture scripts are much easier to understand and modify than JSON or gob +// inputs, and allow for the serialization particulars of any type to be +// independent of the tests. +// +// # Language +// Each line of a script is parsed into a sequence of space-separated command +// words using shell quoting rules, with # marking an end-of-line comment. +// +// Exact semantics depend on the fixture being constructed, but generally +// commands are imperative: commands will affect commands that come after. +// The [CreateIndexReport] example demonstrates how it typically works. +// +// # Implementations +// +// The [Parse] function is generic, but requires specific conventions for the +// types passed in because dispatch happens via [reflect]. See the documentation +// of [Parse] and [Parser]. +package fixturescript + +import ( + "bufio" + "encoding" + "encoding/json" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "unicode" + + "github.com/hugelgupf/go-shlex" +) + +// Parse ... +func Parse[T any, Ctx any](out Parser[*T], pc *Ctx, name string, r io.Reader) (*T, error) { + fv := reflect.ValueOf(out) + // Do some reflect nastiness to make sure "fv" ends up with a pointer in it. +WalkType: + switch fv.Kind() { + case reflect.Pointer: // OK + case reflect.Interface: + fv = fv.Elem().Addr() + goto WalkType + default: + fv = fv.Addr() + goto WalkType + } + // We'll be passing this in to every call, so create it once. + pcv := reflect.ValueOf(pc) + // Use a static list of prefixes to use when constructing a call. + // This allows for shorter names for more common cases. + prefixes := []string{"", "Add", "Push", "Pop"} + + // TODO(hank) This function might be sped up by keeping a global cache for + // this dispatcher construction, keyed by the "Parser" type. + calls := make(map[string]reflect.Value) + ft := fv.Type() + for i := 0; i < ft.NumMethod(); i++ { + m := ft.Method(i) + // Disallow a command of the one method we statically know must be here. + // It's not a real script environment, there's no way to manipulate the returned value. + if m.Name == "Value" { + continue + } + calls[m.Name] = m.Func + } + + s := bufio.NewScanner(r) + s.Split(bufio.ScanLines) + lineNo := 0 + for s.Scan() { + lineNo++ + line, _, _ := strings.Cut(s.Text(), "#") + if len(line) == 0 { + continue + } + + var cmd string + var args []string + if i := strings.IndexFunc(line, unicode.IsSpace); i == -1 { + cmd = line + } else { + cmd = line[:i] + args = shlex.Split(line[i:]) + } + + // Slightly odd construction to try all the prefixes: + // as soon as one name is valid, jump past the error return. + var m reflect.Value + var ok bool + for _, pre := range prefixes { + m, ok = calls[pre+cmd] + if ok { + goto Call + } + } + return nil, fmt.Errorf("%s:%d: unrecognized command %q", name, lineNo, cmd) + + Call: + av := reflect.ValueOf(args) + // Next two lines will panic if not following the convention: + res := m.Call([]reflect.Value{fv, pcv, av}) + if errRet := res[0]; !errRet.IsNil() { + return nil, fmt.Errorf("%s:%d: command %s: %w", name, lineNo, cmd, errRet.Interface().(error)) + } + } + if err := s.Err(); err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + + return out.Value(), nil +} + +// Parser ... +// +// There are additional restrictions on values used as a Parser: +// +// - Any exported methods must have a pointer receiver. +// - Exported methods must accept the "Ctx" type passed to [Parse] as the first argument, +// a slice of strings as the second argument, +// and return an [error]. +type Parser[Out any] interface { + Value() Out +} + +// AssignToStruct is a helper for writing setter commands. +// +// It interprets the "args" array as a key-value pair separated by a "=". +// If the key is the name of a field, the value is interpreted as the type of +// the field and assigned to it. Supported types are: +// +// - int64 +// - int +// - string +// - encoding.TextUnmarshaler +// - json.Unmarshaler +func AssignToStruct[T any](tgt *T, args []string) (err error) { + dv := reflect.ValueOf(tgt).Elem() + for _, arg := range args { + k, v, ok := strings.Cut(arg, "=") + if !ok { + return fmt.Errorf("malformed arg: %q", arg) + } + f := dv.FieldByName(k) + if !f.IsValid() { + return fmt.Errorf("unknown key: %q", k) + } + switch x := f.Addr().Interface(); x := x.(type) { + case *int64: + *x, err = strconv.ParseInt(v, 10, 0) + case *int: + var tmp int64 + tmp, err = strconv.ParseInt(v, 10, 0) + if err == nil { + *x = int(tmp) + } + case *string: + *x = v + case encoding.TextUnmarshaler: + err = x.UnmarshalText([]byte(v)) + case json.Unmarshaler: + err = x.UnmarshalJSON([]byte(v)) + } + if err != nil { + return fmt.Errorf("key %q: bad value %q: %w", k, v, err) + } + } + return nil +} diff --git a/test/fixturescript/indexreport.go b/test/fixturescript/indexreport.go new file mode 100644 index 000000000..2bc792989 --- /dev/null +++ b/test/fixturescript/indexreport.go @@ -0,0 +1,156 @@ +package fixturescript + +import ( + "errors" + "io" + "strconv" + + "github.com/quay/claircore" +) + +// TODO(hank) Should this import `claircore`? Might be cycle concerns. + +// CreateIndexReport ... +// +// Has the following commands: +// +// - AddManifest (sets manifest digest, only allowed once) +// - AddLayer (sets current layer digest) +// - AddDistribution (sets current distribution) +// - ClearDistribution (clears current distribution) +// - PushRepository (pushes a repository onto the repository stack) +// - PopRepository (pops a repository off the repository stack) +// - AddPackage (emits a package using the current Distribution and repository stack) +func CreateIndexReport(name string, r io.Reader) (*claircore.IndexReport, error) { + f := indexReportFixure{ + IndexReport: &claircore.IndexReport{ + Packages: make(map[string]*claircore.Package), + Distributions: make(map[string]*claircore.Distribution), + Repositories: make(map[string]*claircore.Repository), + Environments: make(map[string][]*claircore.Environment), + }, + } + pc := indexReportCtx{} + return Parse(&f, &pc, name, r) +} + +type indexReportFixure struct { + IndexReport *claircore.IndexReport +} + +type indexReportCtx struct { + CurLayer claircore.Digest + CurDistribution *claircore.Distribution + CurSource *claircore.Package + CurPackageDB string + CurRepositoryIDs []string + + ManifestSet bool + LayerSet bool +} + +func (f *indexReportFixure) Value() *claircore.IndexReport { + return f.IndexReport +} + +func (f *indexReportFixure) commonChecks(pc *indexReportCtx, args []string) error { + switch { + case len(args) == 0: + return errors.New("bad number of arguments: want 1 or more") + case !pc.ManifestSet: + return errors.New("bad command: no Manifest created") + case !pc.LayerSet: + return errors.New("bad command: no Layer created") + } + return nil +} + +func (f *indexReportFixure) AddManifest(pc *indexReportCtx, args []string) (err error) { + if len(args) != 1 { + return errors.New("bad number of arguments: want exactly 1") + } + if pc.ManifestSet { + return errors.New("bad command: Manifest already created") + } + f.IndexReport.Hash, err = claircore.ParseDigest(args[0]) + if err != nil { + return err + } + pc.ManifestSet = true + return nil +} + +func (f *indexReportFixure) AddLayer(pc *indexReportCtx, args []string) (err error) { + if len(args) != 1 { + return errors.New("bad number of arguments: want exactly 1") + } + if !pc.ManifestSet { + return errors.New("bad command: no Manifest created") + } + pc.CurLayer, err = claircore.ParseDigest(args[0]) + return err +} + +func (f *indexReportFixure) AddDistribution(pc *indexReportCtx, args []string) error { + f.commonChecks(pc, args) + d := claircore.Distribution{} + if err := AssignToStruct(&d, args); err != nil { + return err + } + pc.CurDistribution = &d + return nil +} + +func (f *indexReportFixure) ClearDistribution(pc *indexReportCtx, args []string) error { + if len(args) == 0 { + return errors.New("bad number of arguments: want 0") + } + pc.CurDistribution = nil + return nil +} + +func (f *indexReportFixure) PushRepository(pc *indexReportCtx, args []string) error { + f.commonChecks(pc, args) + r := claircore.Repository{} + if err := AssignToStruct(&r, args); err != nil { + return err + } + if r.ID == "" { + r.ID = strconv.FormatInt(int64(len(f.IndexReport.Repositories)), 10) + } + f.IndexReport.Repositories[r.ID] = &r + pc.CurRepositoryIDs = append(pc.CurRepositoryIDs, r.ID) + return nil +} + +func (f *indexReportFixure) PopRepository(pc *indexReportCtx, args []string) error { + if len(args) != 0 { + return errors.New("bad number of arguments: want 0") + } + last := len(pc.CurRepositoryIDs) - 1 + pc.CurRepositoryIDs = pc.CurRepositoryIDs[:last:last] // Forces a unique slice when down-sizing. + return nil +} + +func (f *indexReportFixure) AddPackage(pc *indexReportCtx, args []string) error { + f.commonChecks(pc, args) + p := claircore.Package{} + if err := AssignToStruct(&p, args); err != nil { + return err + } + if p.ID == "" { + p.ID = strconv.FormatInt(int64(len(f.IndexReport.Packages)), 10) + } + p.Source = pc.CurSource + f.IndexReport.Packages[p.ID] = &p + env := claircore.Environment{ + PackageDB: p.PackageDB, + IntroducedIn: pc.CurLayer, + RepositoryIDs: pc.CurRepositoryIDs, + } + if pc.CurDistribution != nil { + env.DistributionID = pc.CurDistribution.ID + } + f.IndexReport.Environments[p.ID] = []*claircore.Environment{&env} + return nil +} diff --git a/test/fixturescript/indexreport_test.go b/test/fixturescript/indexreport_test.go new file mode 100644 index 000000000..1f14644bc --- /dev/null +++ b/test/fixturescript/indexreport_test.go @@ -0,0 +1,49 @@ +package fixturescript + +import ( + "fmt" + "sort" + "strings" +) + +func ExampleCreateIndexReport() { + const example = `# Sample IndexReport Fixture +Manifest sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +Layer sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +Repository URI=http://example.com/os-repo +Package Name=hello Version=2.12 PackageDB=bdb:var/lib/rpm +Layer sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +PopRepository +Repository URI=http://example.com/my-repo +Package Name=bash Version=5.2.26 PackageDB=bdb:var/lib/rpm +` + report, err := CreateIndexReport("script", strings.NewReader(example)) + if err != nil { + panic(err) + } + pkgIDs := make([]string, 0, len(report.Packages)) + for id := range report.Packages { + pkgIDs = append(pkgIDs, id) + } + sort.Strings(pkgIDs) + fmt.Println("Manifest:", report.Hash) + for _, id := range pkgIDs { + pkg := report.Packages[id] + fmt.Println("Package:", pkg.Name, pkg.Version) + for _, env := range report.Environments[id] { + fmt.Println("\tLayer:", env.IntroducedIn) + fmt.Println("\tPackage DB:", env.PackageDB) + fmt.Println("\tRepositories:", env.RepositoryIDs) + } + } + // Output: + // Manifest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + // Package: hello 2.12 + // Layer: sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + // Package DB: bdb:var/lib/rpm + // Repositories: [0] + // Package: bash 5.2.26 + // Layer: sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc + // Package DB: bdb:var/lib/rpm + // Repositories: [1] +}