From 988344155a481d5653fa05ecad3758db11eaef2a Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Wed, 9 Sep 2026 10:51:17 -0600 Subject: [PATCH] Add convert to DataBytes and materialize for plain vectors of values - `convert(DataBytes, ::AbstractVector{UInt8})` (and the identity for a `DataBytes`), so `Vector{DataBytes}` and `Vector{Union{Missing, DataBytes}}` accept byte vectors the way `DataString` already accepts strings. - `materialize` on a plain `Vector` of `DataString`/`DataBytes` values (with or without `missing`): copies every element out to an owned `String`/`Vector{UInt8}`, detaching it from the buffers the values reference. Until now it applied only to `StringVector` / `BytesVector` columns; a column a data source handed out (MySQL.jl 2.0 results, for example) had no way to be detached in one call. - Version 1.1.0. Co-Authored-By: Claude Fable 5.1 --- Project.toml | 2 +- README.md | 5 ++++- src/DataStrings.jl | 15 +++++++++++++++ src/construct.jl | 2 ++ test/construct.jl | 28 ++++++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index de29de9..97edba5 100644 --- a/Project.toml +++ b/Project.toml @@ -18,7 +18,7 @@ name = "DataStrings" uuid = "48204bd6-5611-42ea-b167-fc1d713f9be2" authors = ["quinnj "] -version = "1.0.0" +version = "1.1.0" [compat] julia = "1.10" diff --git a/README.md b/README.md index 4a73832..9a9ad01 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,10 @@ Only `DataString` is exported. Use `DataStrings.` for the other public names. `payloadlength`, `payloadbufidx`, `payloadoffset`, `payloadpos`, `INLINE_MAX`, and `PAYLOAD_MISSING`: supported builder interface. - `materialize(column)`: copy the current values into ordinary Julia strings or - byte vectors. Nullable columns preserve missing values. + byte vectors. Nullable columns preserve missing values. It also accepts a plain + `Vector` of `DataString`/`DataBytes` values (with or without `missing`), such as a + column a data source handed out, and detaches the copies from the buffers the + values reference. ## Buffer interface diff --git a/src/DataStrings.jl b/src/DataStrings.jl index de28794..dc1360c 100644 --- a/src/DataStrings.jl +++ b/src/DataStrings.jl @@ -823,6 +823,21 @@ Base.@propagate_inbounds @inline function Base.getindex( return _unchecked_databytes(p, v.buffers[payloadbufidx(p) + 1]) end +""" + materialize(v::AbstractVector{<:Union{Missing,DataString}}) -> Vector{String} or Vector{Union{String,Missing}} + materialize(v::AbstractVector{<:Union{Missing,DataBytes}}) -> Vector{Vector{UInt8}} or Vector{Union{Vector{UInt8},Missing}} + +Copy every element of a plain vector of values (a column a data source handed out, for +example) out to an owned `String` or `Vector{UInt8}`, detaching the result from the buffers +the values reference; `missing` elements are kept. +""" +materialize(v::AbstractVector{DataString}) = String[String(x) for x in v] +materialize(v::AbstractVector{Union{Missing,DataString}}) = + Union{String,Missing}[x === missing ? missing : String(x) for x in v] +materialize(v::AbstractVector{DataBytes}) = Vector{UInt8}[Vector{UInt8}(x) for x in v] +materialize(v::AbstractVector{Union{Missing,DataBytes}}) = + Union{Vector{UInt8},Missing}[x === missing ? missing : Vector{UInt8}(x) for x in v] + """ materialize(v::BytesVector) -> Vector{Vector{UInt8}} or Vector{Union{Vector{UInt8},Missing}} diff --git a/src/construct.jl b/src/construct.jl index a9bed1c..34b1efa 100644 --- a/src/construct.jl +++ b/src/construct.jl @@ -38,6 +38,8 @@ function DataBytes(bytes::AbstractVector{UInt8}) end DataBytes(bytes::DataBytes) = bytes +Base.convert(::Type{DataBytes}, bytes::AbstractVector{UInt8}) = DataBytes(bytes) +Base.convert(::Type{DataBytes}, bytes::DataBytes) = bytes function _append_value!(v::StringVector, s::AbstractString) n = ncodeunits(s) diff --git a/test/construct.jl b/test/construct.jl index 24930e9..c86cc05 100644 --- a/test/construct.jl +++ b/test/construct.jl @@ -73,3 +73,31 @@ @test isequal(AS.materialize(v),oracle) @test isempty(Test.detect_ambiguities(DataStrings, Base; recursive=true)) end + +@testset "convert and materialize plain vectors" begin + b = UInt8[1, 2, 3] + long = collect(UInt8, 1:40) + db = convert(DataBytes, b) + @test db isa DataBytes && db == b + @test convert(DataBytes, db) === db + @test convert(Vector{DataBytes}, [b, long]) == [b, long] + @test isequal(convert(Vector{Union{Missing,DataBytes}}, [b, missing]), [b, missing]) + strs = [DataString("a"), DataString("x"^20)] + @test AS.materialize(strs) isa Vector{String} + @test AS.materialize(strs) == ["a", "x"^20] + withmissing = AS.materialize([DataString("a"), missing]) + @test withmissing isa Vector{Union{String,Missing}} && isequal(withmissing, ["a", missing]) + @test AS.materialize([DataBytes(b), DataBytes(long)]) isa Vector{Vector{UInt8}} + @test AS.materialize([DataBytes(b), DataBytes(long)]) == [b, long] + bytesmissing = AS.materialize([DataBytes(long), missing]) + @test bytesmissing isa Vector{Union{Vector{UInt8},Missing}} && isequal(bytesmissing, [long, missing]) + # the copies are detached from the buffer the views referenced + text = "hello world, this is a long value" + buf = Vector{UInt8}(codeunits(text)) + n = length(buf) + copied_s = AS.materialize([DataString(AS.view_payload(buf, 1, n, 0, 0), buf)])[1] + copied_b = AS.materialize([DataBytes(AS.view_payload(buf, 1, n, 0, 0), buf)])[1] + fill!(buf, 0x00) + @test copied_s == text + @test copied_b == codeunits(text) +end