From 9bfd1a1f8a615a1d8ef2e429a59348fb70ac4645 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 15 Feb 2024 06:22:13 -0500 Subject: [PATCH 1/5] Delete used IOBuffers constant (#1146) --- src/clientlayers/StreamRequest.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/clientlayers/StreamRequest.jl b/src/clientlayers/StreamRequest.jl index 64168a59..4f6e8a9c 100644 --- a/src/clientlayers/StreamRequest.jl +++ b/src/clientlayers/StreamRequest.jl @@ -143,9 +143,6 @@ function readbody(stream::Stream, res::Response, decompress::Union{Nothing, Bool end end -# 2 most common types of IOBuffers -const IOBuffers = Union{IOBuffer, Base.GenericIOBuffer{SubArray{UInt8, 1, Vector{UInt8}, Tuple{UnitRange{Int64}}, true}}} - function readbody!(stream::Stream, res::Response, buf_or_stream, lock) n = 0 if !iserror(res) From 52d2a06a6eeea3ffd0b1d6467a86be1220033aa5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 16 Feb 2024 19:35:09 -0500 Subject: [PATCH 2/5] delete ByteView (#1149) * delete ByteView * Update Streams.jl * Use `isempty` instead equality with an empty buffer --- src/Connections.jl | 4 ++-- src/IOExtras.jl | 7 +++---- src/Streams.jl | 5 +++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Connections.jl b/src/Connections.jl index 60d6eb9b..dd716340 100644 --- a/src/Connections.jl +++ b/src/Connections.jl @@ -234,12 +234,12 @@ Read until `find_delimiter(bytes)` returns non-zero. Return view of bytes up to the delimiter. """ function IOExtras.readuntil(c::Connection, f::F #=Vector{UInt8} -> Int=#, - sizehint=4096)::ByteView where {F <: Function} + sizehint=4096) where {F <: Function} buf = c.buffer if bytesavailable(buf) == 0 read_to_buffer(c, sizehint) end - while (bytes = IOExtras.readuntil(buf, f)) == nobytes + while isempty(begin bytes = IOExtras.readuntil(buf, f) end) read_to_buffer(c, sizehint) end return bytes diff --git a/src/IOExtras.jl b/src/IOExtras.jl index f45127d2..545ef70f 100644 --- a/src/IOExtras.jl +++ b/src/IOExtras.jl @@ -11,7 +11,7 @@ using Sockets using MbedTLS: SSLContext, MbedException using OpenSSL: SSLStream -export bytes, isbytes, nbytes, ByteView, nobytes, +export bytes, isbytes, nbytes, nobytes, startwrite, closewrite, startread, closeread, readuntil, tcpsocket, localport, safe_getpeername @@ -104,7 +104,6 @@ function safe_getpeername(io) end -const ByteView = typeof(view(UInt8[], 1:0)) const nobytes = view(UInt8[], 1:0) readuntil(args...) = Base.readuntil(args...) @@ -114,8 +113,8 @@ Read from an `IO` stream until `find_delimiter(bytes)` returns non-zero. Return view of bytes up to the delimiter. """ function readuntil(buf::IOBuffer, - find_delimiter::F #= Vector{UInt8} -> Int =# - )::ByteView where {F <: Function} + find_delimiter::F #= Vector{UInt8} -> Int =# + ) where {F <: Function} l = find_delimiter(view(buf.data, buf.ptr:buf.size)) if l == 0 return nobytes diff --git a/src/Streams.jl b/src/Streams.jl index 220d11c4..d58e14c8 100644 --- a/src/Streams.jl +++ b/src/Streams.jl @@ -320,14 +320,15 @@ function readall!(http::Stream, buf::Base.GenericIOBuffer=PipeBuffer()) return n end -function IOExtras.readuntil(http::Stream, f::Function)::ByteView +function IOExtras.readuntil(http::Stream, f::Function) UInt(ntoread(http)) == 0 && return Connections.nobytes try bytes = IOExtras.readuntil(http.stream, f) update_ntoread(http, length(bytes)) return bytes - catch + catch ex # if we error, it means we didn't find what we were looking for + # TODO: this seems very sketchy return UInt8[] end end From 43b82b0399fedd9a7e4a7849e7d9cf0b3fc6eea1 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 4 Mar 2024 20:53:32 -0500 Subject: [PATCH 3/5] Update readbytes! to avoid using internal API of IOBuffer less (#1148) * Update readbytes! to avoid using internal API of IOBuffer less Note that this claims to be supported for all GenericIOBuffer, but Base only defines this function on a normal IOBuffer for which `pointer(data)` is contiguous space * Update Streams.jl * Update Streams.jl * Update Streams.jl * fix some tests relying on implementation details --- src/Streams.jl | 19 +++++++------------ test/client.jl | 13 ++++++++----- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/Streams.jl b/src/Streams.jl index d58e14c8..768965bd 100644 --- a/src/Streams.jl +++ b/src/Streams.jl @@ -279,21 +279,15 @@ function Base.unsafe_read(http::Stream, p::Ptr{UInt8}, n::UInt) nothing end -@noinline function bufcheck(buf::Base.GenericIOBuffer, n) - requested_buffer_capacity = (buf.append ? buf.size : (buf.ptr - 1)) + n - requested_buffer_capacity > length(buf.data) && throw(ArgumentError("Unable to grow response stream IOBuffer $(length(buf.data)) large enough for response body size: $requested_buffer_capacity")) -end - function Base.readbytes!(http::Stream, buf::Base.GenericIOBuffer, n=bytesavailable(http)) - Base.ensureroom(buf, n) - # check if there's enough room in buf to write n bytes - bufcheck(buf, n) - data = buf.data - GC.@preserve data unsafe_read(http, pointer(data, (buf.append ? buf.size + 1 : buf.ptr)), n) + p, nbmax = Base.alloc_request(buf, UInt(n)) + nbmax < n && throw(ArgumentError("Unable to grow response stream IOBuffer $nbmax large enough for response body size: $n")) + GC.@preserve buf unsafe_read(http, p, UInt(n)) + # TODO: use `Base.notify_filled(buf, Int(n))` here, but only once it is identical to this: if buf.append - buf.size += n + buf.size += Int(n) else - buf.ptr += n + buf.ptr += Int(n) buf.size = max(buf.size, buf.ptr - 1) end return n @@ -327,6 +321,7 @@ function IOExtras.readuntil(http::Stream, f::Function) update_ntoread(http, length(bytes)) return bytes catch ex + ex isa EOFError || rethrow() # if we error, it means we didn't find what we were looking for # TODO: this seems very sketchy return UInt8[] diff --git a/test/client.jl b/test/client.jl index f9891acf..7251980d 100644 --- a/test/client.jl +++ b/test/client.jl @@ -140,7 +140,7 @@ end # wrapping pre-allocated buffer in IOBuffer will write to buffer directly io = IOBuffer(body; write=true) r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) - @test body === r.body.data + @test Base.mightalias(body, r.body.data) # if provided buffer is too small, we won't grow it for user body = zeros(UInt8, 10) @@ -156,27 +156,30 @@ end # but if you wrap it in a writable IOBuffer, we will grow it io = IOBuffer(body; write=true) r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) - # same Array, though it was resized larger - @test body === r.body.data + # might be a new Array, resized larger + body = take!(io) @test length(body) == 100 # and you can reuse it seekstart(io) r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) - # same Array, though it was resized larger - @test body === r.body.data + # `take!` should have given it a new Array + @test !Base.mightalias(body, r.body.data) + body = take!(io) @test length(body) == 100 # we respect ptr and size body = zeros(UInt8, 100) io = IOBuffer(body; write=true, append=true) # size=100, ptr=1 r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) + body = take!(io) @test length(body) == 200 body = zeros(UInt8, 100) io = IOBuffer(body, write=true, append=false) write(io, body) # size=100, ptr=101 r = HTTP.get("https://$httpbin/bytes/100"; response_stream=io, socket_type_tls=tls) + body = take!(io) @test length(body) == 200 end From 49de8c3de7807ee27e04f014c177cc8e1b267486 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 5 Feb 2025 13:49:18 +0000 Subject: [PATCH 4/5] Add a testset block --- test/websockets/autobahn.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/websockets/autobahn.jl b/test/websockets/autobahn.jl index 8aea00ce..bd3ae730 100644 --- a/test/websockets/autobahn.jl +++ b/test/websockets/autobahn.jl @@ -66,7 +66,7 @@ end end report = JSON.parsefile(joinpath(DIR, "reports/clients/index.json")) - for (k, v) in pairs(report["main"]) + @testset "$k" for (k, v) in pairs(report["main"]) @test v["behavior"] in ("OK", "NON-STRICT", "INFORMATIONAL") end end @@ -96,4 +96,4 @@ end end # @testset "WebSockets" -end # 64-bit only \ No newline at end of file +end # 64-bit only From 3cae52bee9c005ecfab7b6c17c2f97b19104f080 Mon Sep 17 00:00:00 2001 From: Nick Robinson Date: Wed, 5 Feb 2025 14:13:18 +0000 Subject: [PATCH 5/5] fixup! Add a testset block --- test/websockets/autobahn.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/websockets/autobahn.jl b/test/websockets/autobahn.jl index bd3ae730..4080b0ae 100644 --- a/test/websockets/autobahn.jl +++ b/test/websockets/autobahn.jl @@ -65,8 +65,9 @@ end end end - report = JSON.parsefile(joinpath(DIR, "reports/clients/index.json")) - @testset "$k" for (k, v) in pairs(report["main"]) + report = JSON.parsefile(joinpath(DIR, "reports/clients/index.json"))["main"] + @testset "$k" for k in sort(collect(keys(report))) + v = report[k] @test v["behavior"] in ("OK", "NON-STRICT", "INFORMATIONAL") end end