Auto-defuse sup output

book/src/super-sql/operators/fuse.md

@@ -29,7 +29,7 @@ Because all values of the input must be read to compute the fused type,
 ---
 
 _Fuse two records_
-```mdtest-spq
+```mdtest-spq fusion
 # spq
 fuse
 # input
@@ -43,7 +43,7 @@ fusion({a?:_::int64,b?:2},<{b:int64}>)
 ---
 
 _Fuse records with type variation_
-```mdtest-spq
+```mdtest-spq fusion
 # spq
 fuse
 # input
@@ -57,7 +57,7 @@ fuse
 ---
 
 _Fuse records with complex type variation_
-```mdtest-spq {data-layout="stacked"}
+```mdtest-spq fusion {data-layout="stacked"}
 # spq
 fuse
 # input

cli/outputflags/flags.go

@@ -73,6 +73,7 @@ func (f *Flags) SetFormatFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&f.forceBinary, "B", false, "allow Super Binary to be sent to a terminal output")
 	fs.BoolVar(&f.jsonPretty, "J", false, "use formatted JSON output independent of -f option")
 	fs.BoolVar(&f.jsonShortcut, "j", false, "use line-oriented JSON output independent of -f option")
+	fs.BoolVar(&f.SUP.Fusion, "fusion", false, "display fusion values (fusion values are otherwise auto-defused)")
 	fs.BoolVar(&f.supPretty, "S", false, "use formatted Super JSON output independent of -f option")
 	fs.BoolVar(&f.supShortcut, "s", false, "use line-oriented Super JSON output independent of -f option")
 }

db/ztests/meta.yaml

@@ -7,7 +7,7 @@ script: |
   super db load -q -use poolB b.sup
   super db -S -c 'from :pools | drop id | sort name | drop ts'
   echo ===
-  super db -S -c 'from poolA@main:objects | {nameof:nameof(this),...this} | drop id'
+  super db -fusion -S -c 'from poolA@main:objects | {nameof:nameof(this),...this} | drop id'
   super db -S -c 'from poolA:log | cut nameof(this)'
 
 inputs:

mdtest/mdtest.go

@@ -233,11 +233,14 @@ func parseMarkdown(source []byte) (map[string]string, []*Test, error) {
 			})
 		case "mdtest-spq":
 			var fails bool
+			var fusion bool
 			var runtime string
 			for _, word := range fcbInfoWords(fcb, source)[1:] {
 				switch {
 				case word == "fails":
 					fails = true
+				case word == "fusion":
+					fusion = true
 				case strings.HasPrefix(word, "runtime="):
 					runtime = strings.TrimPrefix(word, "runtime=")
 					if runtime != "vam" && runtime != "sam" {
@@ -263,6 +266,7 @@ func parseMarkdown(source []byte) (map[string]string, []*Test, error) {
 				Input:    sections[2],
 				SPQ:      sections[1],
 				Runtime:  runtime,
+				Fusion:   fusion,
 			})
 		}
 		return ast.WalkContinue, nil

mdtest/test.go

@@ -23,8 +23,9 @@ type Test struct {
 	Runtime   string // "sam", "vam", or "" for both
 
 	// For SPQ tests
-	Input string
-	SPQ   string
+	Input  string
+	SPQ    string
+	Fusion bool // If true do not auto-defuse output
 }
 
 // Run runs the test, returning nil on success.
@@ -52,6 +53,9 @@ func (t *Test) run(runtime string) error {
 	var c *exec.Cmd
 	if t.SPQ != "" {
 		c = exec.Command("super", "-s", "-c", t.SPQ)
+		if t.Fusion {
+			c.Args = append(c.Args, "-fusion")
+		}
 		if s := t.Input; strings.TrimSpace(s) != "" {
 			c.Args = append(c.Args, "-")
 			c.Stdin = strings.NewReader(s)

runtime/sam/expr/function/parse.go

@@ -4,10 +4,9 @@ import (
 	"fmt"
 	"net/url"
 	"strconv"
-	"strings"
 
 	"github.com/brimdata/super"
-	"github.com/brimdata/super/sio/supio"
+	"github.com/brimdata/super/pkg/byteconv"
 	"github.com/brimdata/super/sup"
 )
 
@@ -86,13 +85,10 @@ func (p *ParseURI) Call(args []super.Value) super.Value {
 
 type ParseSUP struct {
 	sctx *super.Context
-	sr   *strings.Reader
-	zr   *supio.Reader
 }
 
 func newParseSUP(sctx *super.Context) *ParseSUP {
-	var sr strings.Reader
-	return &ParseSUP{sctx, &sr, supio.NewReader(sctx, &sr)}
+	return &ParseSUP{sctx}
 }
 
 func (p *ParseSUP) Call(args []super.Value) super.Value {
@@ -103,13 +99,9 @@ func (p *ParseSUP) Call(args []super.Value) super.Value {
 	if !in.IsString() {
 		return p.sctx.WrapError("parse_sup: string arg required", args[0])
 	}
-	p.sr.Reset(super.DecodeString(in.Bytes()))
-	val, err := p.zr.Read()
+	val, err := sup.ParseValue(p.sctx, byteconv.UnsafeString(in.Bytes()))
 	if err != nil {
 		return p.sctx.WrapError("parse_sup: "+err.Error(), args[0])
 	}
-	if val == nil {
-		return super.Null
-	}
-	return *val
+	return val
 }

runtime/ztests/expr/function/upcast.yaml

@@ -12,6 +12,8 @@ output: |
 spq: |
   upcast(this[0], this[1])
 
+input-flags: -fusion
+
 input: |
   [[1,"a"],<[int8|string]>]
   [[1::int8,"a"],<[int8|string]>]

runtime/ztests/expr/fusion-all.yaml

@@ -1,5 +1,7 @@
 spq: upcast(this,<fusion(all)>)
 
+input-flags: -fusion
+
 # note: the syntax for fusion values will change from bytes to values soon
 # and this test will be updated
 

runtime/ztests/op/fuse.yaml

@@ -1,5 +1,7 @@
 spq: fuse
 
+input-flags: -fusion
+
 input: |
   {a:"hello",b:"world"}
   {b:"goodnight",c:"gracie"}
@@ -17,6 +19,8 @@ output: |
 # Test fuse on an array of records.
 spq: fuse
 
+input-flags: -fusion
+
 input: |
   [{a:1}]
   [{b:2}]
@@ -36,6 +40,8 @@ output: |
 # Test fuse on mix of complex and primitive values.
 spq: fuse
 
+input-flags: -fusion
+
 input: |
   {a:1}
   {a:"s"}
@@ -57,6 +63,8 @@ output: |
 # Test fuse on nested records.
 spq: fuse
 
+input-flags: -fusion
+
 input: |
   {a:"hello",r:{x:1::int32,y:2::int32}}
   {r:{y:4::int32,z:5::int32},s:"world",r2:{x:6::int32}}
@@ -71,6 +79,8 @@ output: |
 
 spq: fuse
 
+input-flags: -fusion
+
 input: |
   error(1)
   error("s")
@@ -83,6 +93,8 @@ output: |
 
 spq: fuse
 
+input-flags: -fusion
+
 input: |
   [1,2]
   ["foo"]
@@ -97,6 +109,8 @@ output: |
 
 spq: fuse
 
+input-flags: -fusion
+
 input: |
   {a:[1,2]}
   {a:["foo"]}
@@ -111,6 +125,8 @@ output: |
 
 spq: fuse
 
+input-flags: -fusion
+
 input: |
   type p1=int64
   1::p1

sio/supio/writer.go

@@ -4,6 +4,8 @@ import (
 	"io"
 
 	"github.com/brimdata/super"
+	"github.com/brimdata/super/runtime/sam/expr"
+	"github.com/brimdata/super/runtime/sam/expr/function"
 	"github.com/brimdata/super/sbuf"
 	"github.com/brimdata/super/sup"
 	"github.com/brimdata/super/vector"
@@ -12,17 +14,24 @@ import (
 type Writer struct {
 	writer    io.WriteCloser
 	formatter *sup.StreamFormatter
+	defuse    expr.Function
 }
 
 type WriterOpts struct {
 	ColorDisabled bool
+	Fusion        bool
 	Pretty        int
 }
 
 func NewWriter(w io.WriteCloser, opts WriterOpts) *Writer {
+	var defuse expr.Function
+	if !opts.Fusion {
+		defuse = function.NewDefuse(super.NewContext())
+	}
 	return &Writer{
 		formatter: sup.NewStreamFormatter(opts.Pretty, opts.ColorDisabled),
 		writer:    w,
+		defuse:    defuse,
 	}
 }
 
@@ -35,6 +44,9 @@ func (w *Writer) Close() error {
 }
 
 func (w *Writer) Write(val super.Value) error {
+	if w.defuse != nil {
+		val = w.defuse.Call([]super.Value{val})
+	}
 	if _, err := io.WriteString(w.writer, w.formatter.FormatValue(val)); err != nil {
 		return err
 	}