change type none to type missing and improve type safety

book/src/super-sql/functions/types/unblend.md

@@ -49,5 +49,5 @@ blend | unblend(this)
 # expected output
 {x:1}
 {x:2,y:3}
-{x:4}
+{x:4,z?:none::int64}
 ```

book/src/super-sql/operators/blend.md

@@ -35,8 +35,8 @@ blend
 {a:1}
 {b:2}
 # expected output
-{a:1::(int64|none),b:none::(int64|none)}
-{a:none::(int64|none),b:2::(int64|none)}
+{a:1::(int64|missing),b:none::(int64|missing)}
+{a:none::(int64|missing),b:2::(int64|missing)}
 ```
 
 ---
@@ -63,6 +63,6 @@ blend
 {a:[1,2]}
 {a:["foo","bar"],b:10.0.0.1}
 # expected output
-{a:[1,2]::[int64|string],b:none::(ip|none)}
-{a:["foo","bar"]::[int64|string],b:10.0.0.1::(ip|none)}
+{a:[1,2]::[int64|string],b:none::(ip|missing)}
+{a:["foo","bar"]::[int64|string],b:10.0.0.1::(ip|missing)}
 ```

book/src/super-sql/operators/fuse.md

@@ -36,8 +36,8 @@ fuse
 {a:1}
 {b:2}
 # expected output
-{a:fusion(1::(int64|none),<int64>),b:fusion(none::(int64|none),<none>)}
-{a:fusion(none::(int64|none),<none>),b:fusion(2::(int64|none),<int64>)}
+{a:fusion(1::(int64|missing),<int64>),b:fusion(none::(int64|missing),<missing>)}
+{a:fusion(none::(int64|missing),<missing>),b:fusion(2::(int64|missing),<int64>)}
 ```
 
 ---
@@ -64,6 +64,6 @@ fuse
 {a:[1,2]}
 {a:["foo","bar"],b:10.0.0.1}
 # expected output
-{a:fusion([fusion(1::(int64|string),<int64>),fusion(2::(int64|string),<int64>)],<[int64]>),b:fusion(none::(ip|none),<none>)}
-{a:fusion([fusion("foo"::(int64|string),<string>),fusion("bar"::(int64|string),<string>)],<[string]>),b:fusion(10.0.0.1::(ip|none),<ip>)}
+{a:fusion([fusion(1::(int64|string),<int64>),fusion(2::(int64|string),<int64>)],<[int64]>),b:fusion(none::(ip|missing),<missing>)}
+{a:fusion([fusion("foo"::(int64|string),<string>),fusion("bar"::(int64|string),<string>)],<[string]>),b:fusion(10.0.0.1::(ip|missing),<ip>)}
 ```

book/src/super-sql/sql/set-ops.md

@@ -184,10 +184,10 @@ fork
 # input
 
 # expected output
-{x:1::(int64|none),y:2::(int64|none),z:none::(int64|none)}
-{x:3::(int64|none),y:4::(int64|none),z:none::(int64|none)}
-{x:5::(int64|none),y:6::(int64|none),z:none::(int64|none)}
-{x:none::(int64|none),y:none::(int64|none),z:2::(int64|none)}
-{x:none::(int64|none),y:none::(int64|none),z:3::(int64|none)}
+{x:1::(int64|missing),y:2::(int64|missing),z:none::(int64|missing)}
+{x:3::(int64|missing),y:4::(int64|missing),z:none::(int64|missing)}
+{x:5::(int64|missing),y:6::(int64|missing),z:none::(int64|missing)}
+{x:none::(int64|missing),y:none::(int64|missing),z:2::(int64|missing)}
+{x:none::(int64|missing),y:none::(int64|missing),z:3::(int64|missing)}
 ```
 ---

book/src/tutorials/shaping.md

@@ -476,9 +476,9 @@ blend
 {y:"foo"}
 {x:2,y:"bar"}
 # expected output
-{x:1::(int64|none),y:none::(string|none)}
-{x:none::(int64|none),y:"foo"::(string|none)}
-{x:2::(int64|none),y:"bar"::(string|none)}
+{x:1::(int64|missing),y:none::(string|missing)}
+{x:none::(int64|missing),y:"foo"::(string|missing)}
+{x:2::(int64|missing),y:"bar"::(string|missing)}
 ```
 
 Whereas a type union for field `x` is produced in the following:
@@ -511,7 +511,7 @@ blend(this)
 {x:"foo",y:"foo"}
 {x:2,y:"bar"}
 # expected output
-<{x:int64|string,y:string|none}>
+<{x:int64|string,y:string|missing}>
 ```
 
 Since the `fuse` here is an aggregate function, it can also be used with

bsup/encoder.go

@@ -140,7 +140,7 @@ func NewPrimitiveEncoder(bctx *Context, vec vector.Any, root bool) Encoder {
 		return NewTypeValueEncoder(bctx, vec)
 	case *vector.Null:
 		return NewNullEncoder(vec.Len())
-	case *vector.None:
+	case *vector.Missing:
 		return NewNoneEncoder(vec.Len())
 	default:
 		panic(fmt.Sprintf("unsupported type in BSUP file: %T", vec))

bsup/loader/none.go

@@ -21,5 +21,5 @@ func (n *none) length() uint32 {
 func (*none) unmarshal(*bsup.Context, field.Projection) {}
 
 func (n *none) project(loader *FrameLoader, projection field.Projection) vector.Any {
-	return vector.NewNone(n.meta.Count)
+	return vector.NewMissing(n.meta.Count)
 }

bsup/loader/option.go

@@ -58,13 +58,13 @@ func (o *option) project(loader *FrameLoader, projection field.Projection) vecto
 	}
 	optionType := typ.(*super.TypeOption)
 	vec := o.values.project(loader, projection)
-	if vec.Kind() == vector.KindNone {
-		return vector.NewOption(optionType, vec)
+	if vec.Kind() == vector.KindMissing {
+		return vector.NewOptionNone(optionType, vec.Len())
 	}
-	nones := o.len - vec.Len()
-	if nones == 0 {
-		return vector.NewOption(optionType, vec)
+	noneLen := o.len - vec.Len()
+	if noneLen == 0 {
+		return vector.NewOptionSome(optionType, vec)
 	}
 	tags := o.load(loader)
-	return vector.NewOptionBoth(optionType, tags, vec, vector.NewNone(nones))
+	return vector.NewOption(optionType, tags, vec, vector.NewMissing(noneLen))
 }

bsup/ztests/cat-files.yaml

@@ -24,7 +24,7 @@ outputs:
       {c:3}
       {d:4}
       // ===
-      {Kind:"ColumnFrameFusedType",Type:<{a:fusion(int64|none),b:fusion(int64|none)}>}
-      {Kind:"SuperFrameFusedType",Type:<{a:fusion(int64|none),b:fusion(int64|none)}>}
-      {Kind:"ColumnFrameFusedType",Type:<{c:fusion(int64|none),d:fusion(int64|none)}>}
-      {Kind:"SuperFrameFusedType",Type:<{c:fusion(int64|none),d:fusion(int64|none)}>}
+      {Kind:"ColumnFrameFusedType",Type:<{a:fusion(int64|missing),b:fusion(int64|missing)}>}
+      {Kind:"SuperFrameFusedType",Type:<{a:fusion(int64|missing),b:fusion(int64|missing)}>}
+      {Kind:"ColumnFrameFusedType",Type:<{c:fusion(int64|missing),d:fusion(int64|missing)}>}
+      {Kind:"SuperFrameFusedType",Type:<{c:fusion(int64|missing),d:fusion(int64|missing)}>}

bsup/ztests/issue-7061.yaml

@@ -1,5 +1,22 @@
+# XXX we are unwrapping b here because to pass this test for now,
+# but we need proper type checking to report an error here (need
+# to use ?? etc). I don't think comparison should do auto-unwrap..
+# this is the point of type-safe queries... you should be alerted
+# by the compiler that you're not handling the missing case when you
+# should.  So what we need to do in a subsequent PR is propagate
+# errors for this condition through the where clause and make it
+# obvious that you need an unwrap or ??.  we could also have a
+# comparison that does unwrap like javascript, === (strong equality)
+# vs == (loose equality) so maybe == does auto-unwrap
+
 script: |
   super -o test.bsup -
+  super -s -c 'from test.bsup | where unwrap(b).ok()==1'
+  echo ===
+  # this should work but doesn't (XXX coming subseqeunt PR)
+  super -s -c 'from test.bsup | where b==some(1)'
+  echo ===
+  # there should be an error here
   super -s -c 'from test.bsup | where b==1'
 
 inputs:
@@ -12,3 +29,5 @@ outputs:
   - name: stdout
     data: |
       {a:1,b?:1}
+      ===
+      ===