Asynchronous http.get is very slow.

[lucasmsoares96]

In version v1.3.1 of the HTTP package, the code below ran in approximately 70 seconds. After updating the package to version 1.9.14, the execution time became unacceptable. I've waited for 5 minutes and the execution hasn't finished.

Environment:

• Julia 1.9.2
• HTTP.jl 1.9.14

using CSV, HTTP, Base.Threads, HTTP, Tables

const conf = (readtimeout=10, retry=false)

function testHTTP(proxy::String; testURL::String)::Bool
    try
        resp::HTTP.Messages.Response = HTTP.get(testURL; proxy="http://$proxy", conf...)
        if resp.status == 200
            println(threadid(), "\tok\t", proxy)
            return true
        else
            println(threadid(), "\tbad\t", proxy)
            return false
        end
    catch e
        println(threadid(), "\tfail\t", proxy)
        # println(e)
        return false
    end
end

function asyncfilter(f::Function, i::Vector{String})::Vector{String}
    if length(i) >= 2
        head = i[1]
        tail = i[2:end]
        fun = @async f(head)
        recursion = @async asyncfilter(f, tail)
        if fetch(fun)
            return [head, fetch(recursion)...]
        else
            return [fetch(recursion)...]
        end
    else
        return i
    end
end

function proxyTest(proxies::Vector{String}; testURL::String)::Vector{String}
    chunkSize::Int64 = cld(length(proxies), nthreads())
    chunks::Vector{Vector{String}} = collect(Iterators.partition(proxies, chunkSize))
    result::Vector{String} = []
    tasks::Vector{Task} = []
    sizehint!(tasks, nthreads())
    for chunk in chunks
        push!(
            tasks,
            @spawn asyncfilter(i -> testHTTP(i; testURL=testURL), chunk)
        )
    end
    for task in tasks
        append!(result, fetch(task))
    end
    return result
end

url = "https://paste.debian.net/plain/1251148"
response = HTTP.get(url)

# Convertendo para vetor
allproxies::Vector{String} = split(String(response.body), '\n')[2:end-1]

@time proxyTest(allproxies; testURL="https://www.google.com");

[quinnj]

This should be fixed in the 2.0 release