diff --git a/go.mod b/go.mod index 07f7a56..f2d3099 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,24 @@ go 1.24.8 require ( github.com/miekg/dns v1.1.70 + golang.getoutline.org/sdk/x v0.1.0 golang.org/x/sync v0.19.0 ) require ( + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect + github.com/onsi/ginkgo/v2 v2.12.0 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.48.1 // indirect + go.uber.org/mock v0.4.0 // indirect + golang.getoutline.org/sdk v0.0.21 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.31.0 // indirect golang.org/x/net v0.48.0 // indirect golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/tools v0.40.0 // indirect ) diff --git a/go.sum b/go.sum index d224094..da637d3 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,70 @@ +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/miekg/dns v1.1.70 h1:DZ4u2AV35VJxdD9Fo9fIWm119BsQL5cZU1cQ9s0LkqA= github.com/miekg/dns v1.1.70/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= +github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA= +github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= +github.com/shadowsocks/go-shadowsocks2 v0.1.5 h1:PDSQv9y2S85Fl7VBeOMF9StzeXZyK1HakRm86CUbr28= +github.com/shadowsocks/go-shadowsocks2 v0.1.5/go.mod h1:AGGpIoek4HRno4xzyFiAtLHkOpcoznZEkAccaI/rplM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/things-go/go-socks5 v0.0.5 h1:qvKaGcBkfDrUL33SchHN93srAmYGzb4CxSM2DPYufe8= +github.com/things-go/go-socks5 v0.0.5/go.mod h1:mtzInf8v5xmsBpHZVbIw2YQYhc4K0jRwzfsH64Uh0IQ= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +golang.getoutline.org/sdk v0.0.21 h1:zgtenz5DMbnIPOsuAOHNiWdrri81fHyBxhSfRi6Dk8s= +golang.getoutline.org/sdk v0.0.21/go.mod h1:raUAs4PYbEaT/cLTK6PviiKSh7gjEj7JJczFFFr41zc= +golang.getoutline.org/sdk/x v0.1.0 h1:8ykaCEC8Eoi3h/2MdGW7uaMAt2BWFCRhrSvuJ0Y/IU0= +golang.getoutline.org/sdk/x v0.1.0/go.mod h1:Vw7FWpLbYifHFYbbo0mXOCkhR14d1ADwjiF7uBQKyzM= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/greasereport/main.go b/greasereport/main.go index 70a6156..565cd8f 100644 --- a/greasereport/main.go +++ b/greasereport/main.go @@ -49,6 +49,7 @@ type TestResult struct { HTTPStatus int } +// TODO: Deduplicate this with the unified ECH testing package (internal/echtest). var curlExitCodeNames = map[int]string{ 1: "CURLE_UNSUPPORTED_PROTOCOL", 2: "CURLE_FAILED_INIT", diff --git a/internal/echtest/run.go b/internal/echtest/run.go new file mode 100644 index 0000000..67d1df8 --- /dev/null +++ b/internal/echtest/run.go @@ -0,0 +1,242 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package echtest + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +type TestResult struct { + Domain string + ECHGrease bool + Error string + CurlExitCode int + CurlErrorName string + DNSLookup time.Duration + TCPConnection time.Duration + TLSHandshake time.Duration + ServerTime time.Duration + TotalTime time.Duration + HTTPStatus int + Stderr string +} + +// curlExitCodeNames maps curl exit codes to their CURL_* string representations. +var curlExitCodeNames = map[int]string{ + 1: "CURLE_UNSUPPORTED_PROTOCOL", + 2: "CURLE_FAILED_INIT", + 3: "CURLE_URL_MALFORMAT", + 4: "CURLE_NOT_BUILT_IN", + 5: "CURLE_COULDNT_RESOLVE_PROXY", + 6: "CURLE_COULDNT_RESOLVE_HOST", + 7: "CURLE_COULDNT_CONNECT", + 8: "CURLE_WEIRD_SERVER_REPLY", + 9: "CURLE_REMOTE_ACCESS_DENIED", + 11: "CURLE_FTP_WEIRD_PASV_REPLY", + 13: "CURLE_FTP_WEIRD_227_FORMAT", + 14: "CURLE_FTP_CANT_GET_HOST", + 15: "CURLE_FTP_CANT_RECONNECT", + 17: "CURLE_FTP_COULDNT_SET_TYPE", + 18: "CURLE_PARTIAL_FILE", + 19: "CURLE_FTP_COULDNT_RETR_FILE", + 21: "CURLE_QUOTE_ERROR", + 22: "CURLE_HTTP_RETURNED_ERROR", + 23: "CURLE_WRITE_ERROR", + 25: "CURLE_UPLOAD_FAILED", + 26: "CURLE_READ_ERROR", + 27: "CURLE_OUT_OF_MEMORY", + 28: "CURLE_OPERATION_TIMEDOUT", + 30: "CURLE_FTP_PORT_FAILED", + 31: "CURLE_FTP_COULDNT_USE_REST", + 33: "CURLE_RANGE_ERROR", + 34: "CURLE_HTTP_POST_ERROR", + 35: "CURLE_SSL_CONNECT_ERROR", + 36: "CURLE_BAD_DOWNLOAD_RESUME", + 37: "CURLE_FILE_COULDNT_READ_FILE", + 38: "CURLE_LDAP_CANNOT_BIND", + 39: "CURLE_LDAP_SEARCH_FAILED", + 41: "CURLE_FUNCTION_NOT_FOUND", + 42: "CURLE_ABORTED_BY_CALLBACK", + 43: "CURLE_BAD_FUNCTION_ARGUMENT", + 45: "CURLE_INTERFACE_FAILED", + 47: "CURLE_TOO_MANY_REDIRECTS", + 48: "CURLE_UNKNOWN_OPTION", + 49: "CURLE_TELNET_OPTION_SYNTAX", + 51: "CURLE_PEER_FAILED_VERIFICATION", + 52: "CURLE_GOT_NOTHING", + 53: "CURLE_SSL_ENGINE_NOTFOUND", + 54: "CURLE_SSL_ENGINE_SETFAILED", + 55: "CURLE_SEND_ERROR", + 56: "CURLE_RECV_ERROR", + 58: "CURLE_SSL_CERTPROBLEM", + 59: "CURLE_SSL_CIPHER", + 60: "CURLE_SSL_CACERT", + 61: "CURLE_BAD_CONTENT_ENCODING", + 62: "CURLE_LDAP_INVALID_URL", + 63: "CURLE_FILESIZE_EXCEEDED", + 64: "CURLE_USE_SSL_FAILED", + 65: "CURLE_SEND_FAIL_REWIND", + 66: "CURLE_SSL_ENGINE_INITFAILED", + 67: "CURLE_LOGIN_DENIED", + 68: "CURLE_TFTP_NOTFOUND", + 69: "CURLE_TFTP_PERM", + 70: "CURLE_REMOTE_DISK_FULL", + 71: "CURLE_TFTP_ILLEGAL", + 72: "CURLE_TFTP_UNKNOWNID", + 73: "CURLE_REMOTE_FILE_EXISTS", + 74: "CURLE_TFTP_NOSUCHUSER", + 75: "CURLE_CONV_FAILED", + 76: "CURLE_CONV_REQD", + 77: "CURLE_SSL_CACERT_BADFILE", + 78: "CURLE_REMOTE_FILE_NOT_FOUND", + 79: "CURLE_SSH", + 80: "CURLE_SSL_SHUTDOWN_FAILED", + 81: "CURLE_AGAIN", + 82: "CURLE_SSL_CRL_BADFILE", + 83: "CURLE_SSL_ISSUER_ERROR", + 84: "CURLE_FTP_PRET_FAILED", + 85: "CURLE_RTSP_CSEQ_ERROR", + 86: "CURLE_RTSP_SESSION_ERROR", + 87: "CURLE_FTP_BAD_FILE_LIST", + 88: "CURLE_CHUNK_FAILED", + 89: "CURLE_NO_CONNECTION_AVAILABLE", + 90: "CURLE_SSL_PINNEDPUBKEYNOTMATCH", + 91: "CURLE_SSL_INVALIDCERTSTATUS", + 92: "CURLE_HTTP2_STREAM", + 93: "CURLE_RECURSIVE_API_CALL", + 94: "CURLE_AUTH_ERROR", + 95: "CURLE_HTTP3", + 96: "CURLE_QUIC_CONNECT_ERROR", +} + +// Run executes a curl command against the specified domain. +func Run( + curlPath string, + domain string, + echGrease bool, + maxTime time.Duration, + proxyURL string, + proxyHeaders []string, +) TestResult { + result := TestResult{ + Domain: domain, + ECHGrease: echGrease, + } + + targetURL := "https://" + domain + + args := []string{ + "-w", + "dnslookup:%{time_namelookup},tcpconnect:%{time_connect},tlsconnect:%{time_appconnect},servertime:%{time_starttransfer},total:%{time_total},httpstatus:%{http_code}", + "--head", + "--max-time", + strconv.FormatFloat(maxTime.Seconds(), 'f', -1, 64), + } + + // Handle proxy options + if proxyURL != "" { + args = append(args, "--proxy", proxyURL) + for _, h := range proxyHeaders { + args = append(args, "--proxy-header", h) + } + // If using a proxy with headers, we usually need verbose mode to see the proxy response. + // If proxy headers are provided, we assume the caller wants to read them from stderr. + if len(proxyHeaders) > 0 { + args = append(args, "-v") + } else { + args = append(args, "-s") + } + } else { + args = append(args, "-s") + } + + if echGrease { + args = append(args, "--ech", "grease") + } else { + args = append(args, "--ech", "false") + } + args = append(args, targetURL) + + cmd := exec.Command(curlPath, args...) + + // Setup environment for custom curl (matching internal/curl/runner.go) + binDir := filepath.Dir(curlPath) + libDir := filepath.Join(filepath.Dir(binDir), "lib") + if libStat, err := os.Stat(libDir); err == nil && libStat.IsDir() { + cmd.Env = append(os.Environ(), "LD_LIBRARY_PATH="+libDir) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + result.Stderr = stderr.String() // Always capture stderr for caller + + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + result.CurlExitCode = exitError.ExitCode() + result.CurlErrorName = curlExitCodeNames[result.CurlExitCode] + } else { + result.Error = fmt.Sprintf("failed to execute curl: %v", err) + return result + } + } else { + // Even if err is nil, there might be curl-level errors recorded in stderr + // that the caller might be interested in, though standard execution succeeded. + } + + // parse the stdout stats + parts := strings.SplitSeq(stdout.String(), ",") + for part := range parts { + kv := strings.Split(part, ":") + if len(kv) != 2 { + continue + } + key := kv[0] + value := kv[1] + + switch key { + case "dnslookup": + f, _ := strconv.ParseFloat(value, 64) + result.DNSLookup = time.Duration(f * float64(time.Second)) + case "tcpconnect": + f, _ := strconv.ParseFloat(value, 64) + result.TCPConnection = time.Duration(f * float64(time.Second)) + case "tlsconnect": + f, _ := strconv.ParseFloat(value, 64) + result.TLSHandshake = time.Duration(f * float64(time.Second)) + case "servertime": + f, _ := strconv.ParseFloat(value, 64) + result.ServerTime = time.Duration(f * float64(time.Second)) + case "total": + f, _ := strconv.ParseFloat(value, 64) + result.TotalTime = time.Duration(f * float64(time.Second)) + case "httpstatus": + i, _ := strconv.Atoi(value) + result.HTTPStatus = i + } + } + + return result +} diff --git a/internal/soax/soax.go b/internal/soax/soax.go new file mode 100644 index 0000000..dd24910 --- /dev/null +++ b/internal/soax/soax.go @@ -0,0 +1,139 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package soax + +import ( + "context" + "fmt" + "math/rand" + "net/url" + "strconv" + "strings" + + "golang.getoutline.org/sdk/x/soax" +) + +// Config holds the credentials and endpoint configuration for the SOAX service. +type Config struct { + APIKey string + PackageKey string + PackageID string + ProxyHost string + ProxyPort int +} + +// NewConfig creates a new Config, validating required fields and setting defaults. +func NewConfig(apiKey, packageKey, packageID, proxyHost, proxyPortStr string) (*Config, error) { + if apiKey == "" { + return nil, fmt.Errorf("API key is required") + } + if packageKey == "" { + return nil, fmt.Errorf("package key is required") + } + if packageID == "" { + return nil, fmt.Errorf("package ID is required") + } + if proxyHost == "" { + proxyHost = "proxy.soax.com" + } + + proxyPort := 5000 + if proxyPortStr != "" { + p, err := strconv.Atoi(proxyPortStr) + if err != nil { + return nil, fmt.Errorf("invalid proxy port %q: %v", proxyPortStr, err) + } + proxyPort = p + } + + return &Config{ + APIKey: apiKey, + PackageKey: packageKey, + PackageID: packageID, + ProxyHost: proxyHost, + ProxyPort: proxyPort, + }, nil +} + +// ListISPs retrieves a list of available ISP operators for the specified country code. +// It combines both mobile and residential ISPs using the provided SDK client. +func ListISPs(cfg *Config, countryISO string) ([]string, error) { + sdkClient := &soax.Client{ + APIKey: cfg.APIKey, + PackageKey: cfg.PackageKey, + } + + ctx := context.Background() + ispMap := make(map[string]bool) + + if mIsps, err := sdkClient.GetMobileISPs(ctx, countryISO, "", ""); err == nil { + for _, isp := range mIsps { + ispMap[isp] = true + } + } + + if rIsps, err := sdkClient.GetResidentialISPs(ctx, countryISO, "", ""); err == nil { + for _, isp := range rIsps { + ispMap[isp] = true + } + } + + if len(ispMap) == 0 { + return nil, fmt.Errorf("no ISPs found for country %s", countryISO) + } + + var isps []string + for isp := range ispMap { + isps = append(isps, isp) + } + return isps, nil +} + +// BuildWebProxyURL constructs an authenticated HTTPS proxy URL for a specific country and ISP. +// An optional sessionID can be provided for sticky sessions; if empty, a random one is generated. +func BuildWebProxyURL(cfg *Config, countryISO, ispName, sessionID string) string { + if sessionID == "" { + sessionID = generateRandomString(10) + } + + params := []string{"package", cfg.PackageID} + if countryISO != "" { + params = append(params, "country", strings.ToLower(countryISO)) + } + if ispName != "" { + params = append(params, "isp", strings.ToLower(ispName)) + } + if sessionID != "" { + params = append(params, "sessionid", sessionID) + } + params = append(params, "sessionlength", "300") + + u := &url.URL{ + Scheme: "https", + User: url.UserPassword(strings.Join(params, "-"), cfg.PackageKey), + Host: fmt.Sprintf("%s:%d", cfg.ProxyHost, cfg.ProxyPort), + } + + return u.String() +} + +func generateRandomString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} diff --git a/soaxreport/README.md b/soaxreport/README.md new file mode 100644 index 0000000..0338d7b --- /dev/null +++ b/soaxreport/README.md @@ -0,0 +1,92 @@ +# SOAX ECH GREASE Report Generation + +This tool tests ECH GREASE compatibility by issuing requests via SOAX proxies. +It iterates through a list of countries and ISPs, running tests with and without +ECH GREASE to simulate diverse network vantage points. + +## Requirements + +You need to build the ECH-enabled `curl` and place it in the workspace directory. See [instructions](../curl/README.md). + +You also need to set the SOAX credentials as environment variables and provide a list of ISO country codes. + +### Configuration + +**SOAX Credentials (Environment Variables)** + +Set the following environment variables with your SOAX API details: + +```bash +export SOAX_API_KEY="YOUR_API_KEY" +export SOAX_PACKAGE_KEY="YOUR_PACKAGE_KEY" +export SOAX_PACKAGE_ID="YOUR_PACKAGE_ID" +# Optional overrides: +# export SOAX_PROXY_HOST="proxy.soax.com" +# export SOAX_PROXY_PORT="5000" +``` + +**Country List (`countries.csv`)** + +The countries file should be a CSV file containing country names and their 2-letter ISO codes. Lines starting with `#` are ignored. + +```csv +"United States",US +"United Kingdom",GB +"Germany",DE +# Add more countries as needed +"Virgin Islands, U.S.",VI +``` + +You can download a complete list of country codes from [here](https://raw.githubusercontent.com/datasets/country-list/master/data.csv). + +## Running + +To run the tool, ensure your environment variables are set, then use the `go run` command from the project root directory: + +```sh +go run ./soaxreport --targetDomain www.google.com +``` + +This will: + +1. Load the SOAX credentials from the environment and the country list (`./workspace/countries.csv` by default). +2. For each country, fetch the list of available ISPs. +3. For each ISP, issue requests to the target domain via a SOAX proxy, once with ECH GREASE and once without. +4. Save the results to `./workspace/soax-results--countries.csv`. + +### Parameters + +* `-workspace `: Directory to store intermediate files. Defaults to `./workspace`. +* `-countries `: Path to CSV file containing country names and ISO codes. Defaults to `./workspace/countries.csv`. +* `-targetDomain `: Target domain to test. Defaults to `www.google.com`. +* `-parallelism `: Maximum number of parallel requests. Defaults to `16`. +* `-verbose`: Enable verbose logging. +* `-maxTime `: Maximum time per curl request. Defaults to `30s`. +* `-curl `: Path to the ECH-enabled curl binary. Defaults to `./workspace/output/bin/curl`. + +### Output Format + +The tool generates two output files in the workspace directory: + +1. **Results CSV** (`workspace/soax-results--countries.csv`): Contains the detailed test results for each request. +2. **ISP Audit Log** (`workspace/soax-isps-audit.json`): A JSON file mapping each country code to the list of ISPs discovered and used during the test. This is useful for auditing coverage. + +The CSV file contains the following columns: + +* `domain`: The domain that was tested. +* `country_code`: The 2-letter ISO country code. +* `country_name`: The full name of the country. +* `isp`: The ISP name of the proxy used. +* `asn`: The ASN of the proxy exit node. +* `exit_node_ip`: The IP address of the proxy exit node. +* `exit_node_isp`: The ISP name reported by the proxy exit node (from headers). +* `ech_grease`: `true` if ECH GREASE was enabled for the request, `false` otherwise. +* `error`: Any error that occurred during the request. +* `curl_exit_code`: The exit code returned by the `curl` command. +* `curl_error_name`: The human-readable name corresponding to the `curl` exit code. +* `dns_lookup_ms`: The duration of the DNS lookup. +* `tcp_connection_ms`: The duration of the TCP connection. +* `tls_handshake_ms`: The duration of the TLS handshake. +* `server_time_ms`: The time from the end of the TLS handshake to the first byte of the response. +* `total_time_ms`: The total duration of the request. +* `http_status`: The HTTP status code of the response. diff --git a/soaxreport/main.go b/soaxreport/main.go new file mode 100644 index 0000000..fb41cdc --- /dev/null +++ b/soaxreport/main.go @@ -0,0 +1,291 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/csv" + "encoding/json" + "flag" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/Jigsaw-Code/ech-research/internal/echtest" + "github.com/Jigsaw-Code/ech-research/internal/soax" + "github.com/Jigsaw-Code/ech-research/internal/workspace" + "golang.org/x/sync/semaphore" +) + +type TestResult struct { + echtest.TestResult + Country string + CountryName string + ISP string + ASN string + ExitNodeIP string + ExitNodeISP string +} + +func runSoaxTest( + curlPath string, + domain string, + country string, + countryName string, + isp string, + proxyURL string, + echGrease bool, + maxTime time.Duration, +) TestResult { + headers := []string{"Respond-With: ip,isp,asn"} + res := echtest.Run(curlPath, domain, echGrease, maxTime, proxyURL, headers) + + result := TestResult{ + TestResult: res, + Country: country, + CountryName: countryName, + ISP: isp, + } + + // Parse metadata from Stderr (SOAX specific headers in CONNECT response) + for line := range strings.SplitSeq(res.Stderr, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "< Node-") { + continue + } + kv := strings.SplitN(strings.TrimPrefix(line, "< Node-"), ":", 2) + if len(kv) != 2 { + continue + } + key := strings.ToLower(strings.TrimSpace(kv[0])) + val := strings.TrimSpace(kv[1]) + switch key { + case "asn": + result.ASN = val + case "ip": + result.ExitNodeIP = val + case "isp": + result.ExitNodeISP = val + } + } + + return result +} + +type Country struct { + Name string + Code string +} + +func loadCountries(path string) ([]Country, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + var countries []Country + reader := csv.NewReader(f) + reader.Comment = '#' // Support skipping lines starting with # + reader.FieldsPerRecord = 2 + + records, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("failed to read countries CSV: %w", err) + } + + for i, record := range records { + name := strings.TrimSpace(record[0]) + code := strings.TrimSpace(record[1]) + + // Skip header row if present + if i == 0 && strings.EqualFold(name, "Name") && strings.EqualFold(code, "Code") { + continue + } + + countries = append(countries, Country{ + Name: name, + Code: code, + }) + } + return countries, nil +} + +func main() { + var ( + workspaceFlag = flag.String("workspace", "./workspace", "Directory to store intermediate files") + countriesFlag = flag.String("countries", "", "Path to file containing ISO country codes") + targetDomainFlag = flag.String("targetDomain", "www.google.com", "Target domain to test") + verboseFlag = flag.Bool("verbose", false, "Enable verbose logging") + maxTimeFlag = flag.Duration("maxTime", 30*time.Second, "Maximum time per curl request") + curlPathFlag = flag.String("curl", "", "Path to the ECH-enabled curl binary") + parallelismFlag = flag.Int("parallelism", 16, "Maximum number of parallel requests") + ) + flag.Parse() + + if *verboseFlag { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) + } else { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))) + } + + // Set up workspace directory + workspaceDir := workspace.EnsureWorkspace(*workspaceFlag) + + // Determine curl binary path + curlPath := *curlPathFlag + if curlPath == "" { + curlPath = filepath.Join(workspaceDir, "output", "bin", "curl") + } + + // Load SOAX config from environment variables + cfg, err := soax.NewConfig( + os.Getenv("SOAX_API_KEY"), + os.Getenv("SOAX_PACKAGE_KEY"), + os.Getenv("SOAX_PACKAGE_ID"), + os.Getenv("SOAX_PROXY_HOST"), + os.Getenv("SOAX_PROXY_PORT"), + ) + if err != nil { + slog.Error("Failed to initialize SOAX config from environment", "error", err) + os.Exit(1) + } + + // Load countries + countriesPath := *countriesFlag + if countriesPath == "" { + countriesPath = filepath.Join(workspaceDir, "countries.csv") + } + countries, err := loadCountries(countriesPath) + if err != nil { + slog.Error("Failed to load countries list", "path", countriesPath, "error", err) + os.Exit(1) + } + + // Create output CSV file + sanitizedDomain := strings.ReplaceAll(*targetDomainFlag, ".", "_") + outputFilename := filepath.Join(workspaceDir, fmt.Sprintf("soax-results-%s-countries%d.csv", sanitizedDomain, len(countries))) + outputFile, err := os.Create(outputFilename) + if err != nil { + slog.Error("Failed to create output CSV file", "path", outputFilename, "error", err) + os.Exit(1) + } + defer outputFile.Close() + + resultsCh := make(chan TestResult, 2*len(countries)*(*parallelismFlag)) + + var csvWg sync.WaitGroup + csvWg.Add(1) + go func() { + defer csvWg.Done() + csvWriter := csv.NewWriter(outputFile) + defer csvWriter.Flush() + + header := []string{ + "domain", "country_code", "country_name", "isp", "asn", "exit_node_ip", "exit_node_isp", "ech_grease", "error", + "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", + "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", + } + if err := csvWriter.Write(header); err != nil { + slog.Error("Failed to write CSV header", "error", err) + } + + for r := range resultsCh { + record := []string{ + r.Domain, r.Country, r.CountryName, r.ISP, r.ASN, r.ExitNodeIP, r.ExitNodeISP, strconv.FormatBool(r.ECHGrease), r.Error, + strconv.Itoa(r.CurlExitCode), r.CurlErrorName, + strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), + strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), + strconv.FormatInt(r.TLSHandshake.Milliseconds(), 10), + strconv.FormatInt(r.ServerTime.Milliseconds(), 10), + strconv.FormatInt(r.TotalTime.Milliseconds(), 10), + strconv.Itoa(r.HTTPStatus), + } + if err := csvWriter.Write(record); err != nil { + slog.Error("Failed to write record to CSV", "error", err) + } + } + }() + + domain := *targetDomainFlag + runSessionID := time.Now().Format("0102150405") + sem := semaphore.NewWeighted(int64(*parallelismFlag)) + var wg sync.WaitGroup + var total, finished atomic.Int32 + + // Audit map to store discovered ISPs per country + ispAuditMap := make(map[string][]string) + + for _, country := range countries { + slog.Debug("Processing country", "name", country.Name, "code", country.Code) + + isps, err := soax.ListISPs(cfg, country.Code) + if err != nil { + slog.Error("Failed to fetch ISPs", "country", country.Code, "error", err) + continue + } + + ispAuditMap[country.Code] = isps + + total.Add(int32(len(isps) * 2)) + for i, isp := range isps { + wg.Add(2) + sessionID := fmt.Sprintf("%s%s%d", runSessionID, country.Code, i) + + startTest := func(c Country, isp, sid string, ech bool) { + defer wg.Done() + if err := sem.Acquire(context.Background(), 1); err != nil { + slog.Error("Failed to acquire semaphore", "error", err) + return + } + defer sem.Release(1) + + proxyURL := soax.BuildWebProxyURL(cfg, c.Code, isp, sid) + slog.Debug("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", ech, "session", sid) + resultsCh <- runSoaxTest(curlPath, domain, c.Code, c.Name, isp, proxyURL, ech, *maxTimeFlag) + progress := fmt.Sprintf("%d/%d", finished.Add(1), total.Load()) + slog.Info("Finished", "country", c.Code, "isp", isp, "progress", progress) + } + + go startTest(country, isp, sessionID, false) + go startTest(country, isp, sessionID, true) + } + } + + wg.Wait() + close(resultsCh) + csvWg.Wait() + + // Write the ISP audit log to JSON + auditFilename := filepath.Join(workspaceDir, "soax-isps-audit.json") + auditData, err := json.MarshalIndent(ispAuditMap, "", " ") + if err == nil { + if err := os.WriteFile(auditFilename, auditData, 0644); err != nil { + slog.Error("Failed to write ISP audit log", "error", err) + } else { + slog.Info("ISP audit log saved", "path", auditFilename) + } + } else { + slog.Error("Failed to marshal ISP audit log", "error", err) + } + + slog.Info("Done. Results saved to", "path", outputFilename) +}