Add convert to DataBytes and materialize for plain vectors of values

Project.toml

@@ -18,7 +18,7 @@
 name = "DataStrings"
 uuid = "48204bd6-5611-42ea-b167-fc1d713f9be2"
 authors = ["quinnj <quinn.jacobd@gmail.com>"]
-version = "1.0.0"
+version = "1.1.0"
 
 [compat]
 julia = "1.10"

README.md

@@ -51,7 +51,10 @@ Only `DataString` is exported. Use `DataStrings.` for the other public names.
   `payloadlength`, `payloadbufidx`, `payloadoffset`, `payloadpos`, `INLINE_MAX`,
   and `PAYLOAD_MISSING`: supported builder interface.
 - `materialize(column)`: copy the current values into ordinary Julia strings or
-  byte vectors. Nullable columns preserve missing values.
+  byte vectors. Nullable columns preserve missing values. It also accepts a plain
+  `Vector` of `DataString`/`DataBytes` values (with or without `missing`), such as a
+  column a data source handed out, and detaches the copies from the buffers the
+  values reference.
 
 ## Buffer interface
 

src/DataStrings.jl

@@ -823,6 +823,21 @@ Base.@propagate_inbounds @inline function Base.getindex(
     return _unchecked_databytes(p, v.buffers[payloadbufidx(p) + 1])
 end
 
+"""
+    materialize(v::AbstractVector{<:Union{Missing,DataString}}) -> Vector{String} or Vector{Union{String,Missing}}
+    materialize(v::AbstractVector{<:Union{Missing,DataBytes}}) -> Vector{Vector{UInt8}} or Vector{Union{Vector{UInt8},Missing}}
+
+Copy every element of a plain vector of values (a column a data source handed out, for
+example) out to an owned `String` or `Vector{UInt8}`, detaching the result from the buffers
+the values reference; `missing` elements are kept.
+"""
+materialize(v::AbstractVector{DataString}) = String[String(x) for x in v]
+materialize(v::AbstractVector{Union{Missing,DataString}}) =
+    Union{String,Missing}[x === missing ? missing : String(x) for x in v]
+materialize(v::AbstractVector{DataBytes}) = Vector{UInt8}[Vector{UInt8}(x) for x in v]
+materialize(v::AbstractVector{Union{Missing,DataBytes}}) =
+    Union{Vector{UInt8},Missing}[x === missing ? missing : Vector{UInt8}(x) for x in v]
+
 """
     materialize(v::BytesVector) -> Vector{Vector{UInt8}} or Vector{Union{Vector{UInt8},Missing}}
 

src/construct.jl

@@ -38,6 +38,8 @@ function DataBytes(bytes::AbstractVector{UInt8})
 end
 
 DataBytes(bytes::DataBytes) = bytes
+Base.convert(::Type{DataBytes}, bytes::AbstractVector{UInt8}) = DataBytes(bytes)
+Base.convert(::Type{DataBytes}, bytes::DataBytes) = bytes
 
 function _append_value!(v::StringVector, s::AbstractString)
     n = ncodeunits(s)

test/construct.jl

@@ -73,3 +73,31 @@
     @test isequal(AS.materialize(v),oracle)
     @test isempty(Test.detect_ambiguities(DataStrings, Base; recursive=true))
 end
+
+@testset "convert and materialize plain vectors" begin
+    b = UInt8[1, 2, 3]
+    long = collect(UInt8, 1:40)
+    db = convert(DataBytes, b)
+    @test db isa DataBytes && db == b
+    @test convert(DataBytes, db) === db
+    @test convert(Vector{DataBytes}, [b, long]) == [b, long]
+    @test isequal(convert(Vector{Union{Missing,DataBytes}}, [b, missing]), [b, missing])
+    strs = [DataString("a"), DataString("x"^20)]
+    @test AS.materialize(strs) isa Vector{String}
+    @test AS.materialize(strs) == ["a", "x"^20]
+    withmissing = AS.materialize([DataString("a"), missing])
+    @test withmissing isa Vector{Union{String,Missing}} && isequal(withmissing, ["a", missing])
+    @test AS.materialize([DataBytes(b), DataBytes(long)]) isa Vector{Vector{UInt8}}
+    @test AS.materialize([DataBytes(b), DataBytes(long)]) == [b, long]
+    bytesmissing = AS.materialize([DataBytes(long), missing])
+    @test bytesmissing isa Vector{Union{Vector{UInt8},Missing}} && isequal(bytesmissing, [long, missing])
+    # the copies are detached from the buffer the views referenced
+    text = "hello world, this is a long value"
+    buf = Vector{UInt8}(codeunits(text))
+    n = length(buf)
+    copied_s = AS.materialize([DataString(AS.view_payload(buf, 1, n, 0, 0), buf)])[1]
+    copied_b = AS.materialize([DataBytes(AS.view_payload(buf, 1, n, 0, 0), buf)])[1]
+    fill!(buf, 0x00)
+    @test copied_s == text
+    @test copied_b == codeunits(text)
+end