diff --git a/book/src/super-sql/operators/fuse.md b/book/src/super-sql/operators/fuse.md index eddd4016f7..72e65b29fe 100644 --- a/book/src/super-sql/operators/fuse.md +++ b/book/src/super-sql/operators/fuse.md @@ -29,7 +29,7 @@ Because all values of the input must be read to compute the fused type, --- _Fuse two records_ -```mdtest-spq +```mdtest-spq fusion # spq fuse # input @@ -43,7 +43,7 @@ fusion({a?:_::int64,b?:2},<{b:int64}>) --- _Fuse records with type variation_ -```mdtest-spq +```mdtest-spq fusion # spq fuse # input @@ -57,7 +57,7 @@ fuse --- _Fuse records with complex type variation_ -```mdtest-spq {data-layout="stacked"} +```mdtest-spq fusion {data-layout="stacked"} # spq fuse # input diff --git a/cli/outputflags/flags.go b/cli/outputflags/flags.go index 0beeeddcf7..86ba0b8bc7 100644 --- a/cli/outputflags/flags.go +++ b/cli/outputflags/flags.go @@ -73,6 +73,7 @@ func (f *Flags) SetFormatFlags(fs *flag.FlagSet) { fs.BoolVar(&f.forceBinary, "B", false, "allow Super Binary to be sent to a terminal output") fs.BoolVar(&f.jsonPretty, "J", false, "use formatted JSON output independent of -f option") fs.BoolVar(&f.jsonShortcut, "j", false, "use line-oriented JSON output independent of -f option") + fs.BoolVar(&f.SUP.Fusion, "fusion", false, "display fusion values (fusion values are otherwise auto-defused)") fs.BoolVar(&f.supPretty, "S", false, "use formatted Super JSON output independent of -f option") fs.BoolVar(&f.supShortcut, "s", false, "use line-oriented Super JSON output independent of -f option") } diff --git a/db/ztests/meta.yaml b/db/ztests/meta.yaml index f861036dbb..6bfdc9c0a0 100644 --- a/db/ztests/meta.yaml +++ b/db/ztests/meta.yaml @@ -7,7 +7,7 @@ script: | super db load -q -use poolB b.sup super db -S -c 'from :pools | drop id | sort name | drop ts' echo === - super db -S -c 'from poolA@main:objects | {nameof:nameof(this),...this} | drop id' + super db -fusion -S -c 'from poolA@main:objects | {nameof:nameof(this),...this} | drop id' super db -S -c 'from poolA:log | cut nameof(this)' inputs: diff --git a/mdtest/mdtest.go b/mdtest/mdtest.go index 037d61d493..4cb3b73d4d 100644 --- a/mdtest/mdtest.go +++ b/mdtest/mdtest.go @@ -233,11 +233,14 @@ func parseMarkdown(source []byte) (map[string]string, []*Test, error) { }) case "mdtest-spq": var fails bool + var fusion bool var runtime string for _, word := range fcbInfoWords(fcb, source)[1:] { switch { case word == "fails": fails = true + case word == "fusion": + fusion = true case strings.HasPrefix(word, "runtime="): runtime = strings.TrimPrefix(word, "runtime=") if runtime != "vam" && runtime != "sam" { @@ -263,6 +266,7 @@ func parseMarkdown(source []byte) (map[string]string, []*Test, error) { Input: sections[2], SPQ: sections[1], Runtime: runtime, + Fusion: fusion, }) } return ast.WalkContinue, nil diff --git a/mdtest/test.go b/mdtest/test.go index 487b25f890..30ed2b2dfc 100644 --- a/mdtest/test.go +++ b/mdtest/test.go @@ -23,8 +23,9 @@ type Test struct { Runtime string // "sam", "vam", or "" for both // For SPQ tests - Input string - SPQ string + Input string + SPQ string + Fusion bool // If true do not auto-defuse output } // Run runs the test, returning nil on success. @@ -52,6 +53,9 @@ func (t *Test) run(runtime string) error { var c *exec.Cmd if t.SPQ != "" { c = exec.Command("super", "-s", "-c", t.SPQ) + if t.Fusion { + c.Args = append(c.Args, "-fusion") + } if s := t.Input; strings.TrimSpace(s) != "" { c.Args = append(c.Args, "-") c.Stdin = strings.NewReader(s) diff --git a/runtime/sam/expr/function/parse.go b/runtime/sam/expr/function/parse.go index 3298644ef0..e4f4cf0019 100644 --- a/runtime/sam/expr/function/parse.go +++ b/runtime/sam/expr/function/parse.go @@ -4,10 +4,9 @@ import ( "fmt" "net/url" "strconv" - "strings" "github.com/brimdata/super" - "github.com/brimdata/super/sio/supio" + "github.com/brimdata/super/pkg/byteconv" "github.com/brimdata/super/sup" ) @@ -86,13 +85,10 @@ func (p *ParseURI) Call(args []super.Value) super.Value { type ParseSUP struct { sctx *super.Context - sr *strings.Reader - zr *supio.Reader } func newParseSUP(sctx *super.Context) *ParseSUP { - var sr strings.Reader - return &ParseSUP{sctx, &sr, supio.NewReader(sctx, &sr)} + return &ParseSUP{sctx} } func (p *ParseSUP) Call(args []super.Value) super.Value { @@ -103,13 +99,9 @@ func (p *ParseSUP) Call(args []super.Value) super.Value { if !in.IsString() { return p.sctx.WrapError("parse_sup: string arg required", args[0]) } - p.sr.Reset(super.DecodeString(in.Bytes())) - val, err := p.zr.Read() + val, err := sup.ParseValue(p.sctx, byteconv.UnsafeString(in.Bytes())) if err != nil { return p.sctx.WrapError("parse_sup: "+err.Error(), args[0]) } - if val == nil { - return super.Null - } - return *val + return val } diff --git a/runtime/ztests/expr/function/upcast.yaml b/runtime/ztests/expr/function/upcast.yaml index 7848eb4a28..2ae6e0e81d 100644 --- a/runtime/ztests/expr/function/upcast.yaml +++ b/runtime/ztests/expr/function/upcast.yaml @@ -12,6 +12,8 @@ output: | spq: | upcast(this[0], this[1]) +input-flags: -fusion + input: | [[1,"a"],<[int8|string]>] [[1::int8,"a"],<[int8|string]>] diff --git a/runtime/ztests/expr/fusion-all.yaml b/runtime/ztests/expr/fusion-all.yaml index e44435bd66..a80a90bc0f 100644 --- a/runtime/ztests/expr/fusion-all.yaml +++ b/runtime/ztests/expr/fusion-all.yaml @@ -1,5 +1,7 @@ spq: upcast(this,) +input-flags: -fusion + # note: the syntax for fusion values will change from bytes to values soon # and this test will be updated diff --git a/runtime/ztests/op/fuse.yaml b/runtime/ztests/op/fuse.yaml index 67884b977e..66955f9662 100644 --- a/runtime/ztests/op/fuse.yaml +++ b/runtime/ztests/op/fuse.yaml @@ -1,5 +1,7 @@ spq: fuse +input-flags: -fusion + input: | {a:"hello",b:"world"} {b:"goodnight",c:"gracie"} @@ -17,6 +19,8 @@ output: | # Test fuse on an array of records. spq: fuse +input-flags: -fusion + input: | [{a:1}] [{b:2}] @@ -36,6 +40,8 @@ output: | # Test fuse on mix of complex and primitive values. spq: fuse +input-flags: -fusion + input: | {a:1} {a:"s"} @@ -57,6 +63,8 @@ output: | # Test fuse on nested records. spq: fuse +input-flags: -fusion + input: | {a:"hello",r:{x:1::int32,y:2::int32}} {r:{y:4::int32,z:5::int32},s:"world",r2:{x:6::int32}} @@ -71,6 +79,8 @@ output: | spq: fuse +input-flags: -fusion + input: | error(1) error("s") @@ -83,6 +93,8 @@ output: | spq: fuse +input-flags: -fusion + input: | [1,2] ["foo"] @@ -97,6 +109,8 @@ output: | spq: fuse +input-flags: -fusion + input: | {a:[1,2]} {a:["foo"]} @@ -111,6 +125,8 @@ output: | spq: fuse +input-flags: -fusion + input: | type p1=int64 1::p1 diff --git a/sio/supio/writer.go b/sio/supio/writer.go index dfb2e392db..34b8bdd2a8 100644 --- a/sio/supio/writer.go +++ b/sio/supio/writer.go @@ -4,6 +4,8 @@ import ( "io" "github.com/brimdata/super" + "github.com/brimdata/super/runtime/sam/expr" + "github.com/brimdata/super/runtime/sam/expr/function" "github.com/brimdata/super/sbuf" "github.com/brimdata/super/sup" "github.com/brimdata/super/vector" @@ -12,17 +14,24 @@ import ( type Writer struct { writer io.WriteCloser formatter *sup.StreamFormatter + defuse expr.Function } type WriterOpts struct { ColorDisabled bool + Fusion bool Pretty int } func NewWriter(w io.WriteCloser, opts WriterOpts) *Writer { + var defuse expr.Function + if !opts.Fusion { + defuse = function.NewDefuse(super.NewContext()) + } return &Writer{ formatter: sup.NewStreamFormatter(opts.Pretty, opts.ColorDisabled), writer: w, + defuse: defuse, } } @@ -35,6 +44,9 @@ func (w *Writer) Close() error { } func (w *Writer) Write(val super.Value) error { + if w.defuse != nil { + val = w.defuse.Call([]super.Value{val}) + } if _, err := io.WriteString(w.writer, w.formatter.FormatValue(val)); err != nil { return err }