From 00c9f8de831df2228141469c5a81ea6bcbd1f89b Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 20 Jan 2026 18:21:56 -0500 Subject: [PATCH 01/19] feat: add `soaxreport` tool for ECH testing via SOAX proxies --- soaxreport/soax.go | 113 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 soaxreport/soax.go diff --git a/soaxreport/soax.go b/soaxreport/soax.go new file mode 100644 index 0000000..03079f5 --- /dev/null +++ b/soaxreport/soax.go @@ -0,0 +1,113 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "os" + "strings" +) + +// SoaxConfig holds the credentials and endpoint configuration for the SOAX service. +type SoaxConfig struct { + APIKey string `json:"api_key"` + PackageKey string `json:"package_key"` + PackageID string `json:"package_id"` + ProxyHost string `json:"proxy_host"` + ProxyPort int `json:"proxy_port"` +} + +// SoaxClient provides methods to interact with the SOAX API and generate proxy configurations. +type SoaxClient struct { + Config *SoaxConfig +} + +// NewSoaxClient creates a new SoaxClient by loading configuration from a JSON file. +// If ProxyHost or ProxyPort are missing in the config, default values are used. +func NewSoaxClient(configPath string) (*SoaxClient, error) { + f, err := os.Open(configPath) + if err != nil { + return nil, fmt.Errorf("failed to open config file: %w", err) + } + defer f.Close() + + var cfg SoaxConfig + if err := json.NewDecoder(f).Decode(&cfg); err != nil { + return nil, fmt.Errorf("failed to decode config json: %w", err) + } + if cfg.ProxyHost == "" { + cfg.ProxyHost = "proxy.soax.com" + } + if cfg.ProxyPort == 0 { + cfg.ProxyPort = 5000 + } + + return &SoaxClient{Config: &cfg}, nil +} + +// ListISPs retrieves a list of available ISP operators for the specified country code. +// countryISO should be a 2-letter ISO country code (e.g., "US"). +func (s *SoaxClient) ListISPs(countryISO string) ([]string, error) { + url := fmt.Sprintf("https://api.soax.com/api/get-country-operators?api_key=%s&package_key=%s&country_iso=%s", + s.Config.APIKey, s.Config.PackageKey, strings.ToLower(countryISO)) + + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to fetch ISPs: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("API error: status %s, body: %s", resp.Status, string(body)) + } + + var isps []string + if err := json.NewDecoder(resp.Body).Decode(&isps); err != nil { + return nil, fmt.Errorf("failed to decode ISP list: %w", err) + } + return isps, nil +} + +// BuildProxyURL constructs an authenticated HTTPS proxy URL for a specific country and ISP. +// An optional sessionID can be provided for sticky sessions; if empty, a random one is generated. +func (s *SoaxClient) BuildProxyURL(countryISO, ispName, sessionID string) string { + if sessionID == "" { + sessionID = generateRandomString(10) + } + + ispName = url.QueryEscape(strings.ToLower(ispName)) + countryISO = strings.ToLower(countryISO) + + proxyUser := fmt.Sprintf("package-%s-country-%s-isp-%s-sessionid-%s-sessionlength-300", + s.Config.PackageID, countryISO, ispName, sessionID) + + return fmt.Sprintf("https://%s:%s@%s:%d", + proxyUser, s.Config.PackageKey, s.Config.ProxyHost, s.Config.ProxyPort) +} + +func generateRandomString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} From 32ed53af41191c3bb7ade7080de32197d483ee9d Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Wed, 21 Jan 2026 16:43:25 -0500 Subject: [PATCH 02/19] Refactor SOAX client into `internal/soax` --- {soaxreport => internal/soax}/soax.go | 37 +++++++++++++++------------ 1 file changed, 21 insertions(+), 16 deletions(-) rename {soaxreport => internal/soax}/soax.go (76%) diff --git a/soaxreport/soax.go b/internal/soax/soax.go similarity index 76% rename from soaxreport/soax.go rename to internal/soax/soax.go index 03079f5..264ff81 100644 --- a/soaxreport/soax.go +++ b/internal/soax/soax.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package soax import ( "encoding/json" @@ -25,8 +25,8 @@ import ( "strings" ) -// SoaxConfig holds the credentials and endpoint configuration for the SOAX service. -type SoaxConfig struct { +// Config holds the credentials and endpoint configuration for the SOAX service. +type Config struct { APIKey string `json:"api_key"` PackageKey string `json:"package_key"` PackageID string `json:"package_id"` @@ -34,21 +34,21 @@ type SoaxConfig struct { ProxyPort int `json:"proxy_port"` } -// SoaxClient provides methods to interact with the SOAX API and generate proxy configurations. -type SoaxClient struct { - Config *SoaxConfig +// Client provides methods to interact with the SOAX API and generate proxy configurations. +type Client struct { + cfg *Config } -// NewSoaxClient creates a new SoaxClient by loading configuration from a JSON file. +// LoadConfig reads the SOAX configuration from a JSON file. // If ProxyHost or ProxyPort are missing in the config, default values are used. -func NewSoaxClient(configPath string) (*SoaxClient, error) { - f, err := os.Open(configPath) +func LoadConfig(path string) (*Config, error) { + f, err := os.Open(path) if err != nil { return nil, fmt.Errorf("failed to open config file: %w", err) } defer f.Close() - var cfg SoaxConfig + var cfg Config if err := json.NewDecoder(f).Decode(&cfg); err != nil { return nil, fmt.Errorf("failed to decode config json: %w", err) } @@ -59,14 +59,19 @@ func NewSoaxClient(configPath string) (*SoaxClient, error) { cfg.ProxyPort = 5000 } - return &SoaxClient{Config: &cfg}, nil + return &cfg, nil +} + +// NewClient creates a new SOAX Client with the given configuration. +func NewClient(cfg *Config) *Client { + return &Client{cfg: cfg} } // ListISPs retrieves a list of available ISP operators for the specified country code. // countryISO should be a 2-letter ISO country code (e.g., "US"). -func (s *SoaxClient) ListISPs(countryISO string) ([]string, error) { +func (c *Client) ListISPs(countryISO string) ([]string, error) { url := fmt.Sprintf("https://api.soax.com/api/get-country-operators?api_key=%s&package_key=%s&country_iso=%s", - s.Config.APIKey, s.Config.PackageKey, strings.ToLower(countryISO)) + c.cfg.APIKey, c.cfg.PackageKey, strings.ToLower(countryISO)) resp, err := http.Get(url) if err != nil { @@ -88,7 +93,7 @@ func (s *SoaxClient) ListISPs(countryISO string) ([]string, error) { // BuildProxyURL constructs an authenticated HTTPS proxy URL for a specific country and ISP. // An optional sessionID can be provided for sticky sessions; if empty, a random one is generated. -func (s *SoaxClient) BuildProxyURL(countryISO, ispName, sessionID string) string { +func (c *Client) BuildProxyURL(countryISO, ispName, sessionID string) string { if sessionID == "" { sessionID = generateRandomString(10) } @@ -97,10 +102,10 @@ func (s *SoaxClient) BuildProxyURL(countryISO, ispName, sessionID string) string countryISO = strings.ToLower(countryISO) proxyUser := fmt.Sprintf("package-%s-country-%s-isp-%s-sessionid-%s-sessionlength-300", - s.Config.PackageID, countryISO, ispName, sessionID) + c.cfg.PackageID, countryISO, ispName, sessionID) return fmt.Sprintf("https://%s:%s@%s:%d", - proxyUser, s.Config.PackageKey, s.Config.ProxyHost, s.Config.ProxyPort) + proxyUser, c.cfg.PackageKey, c.cfg.ProxyHost, c.cfg.ProxyPort) } func generateRandomString(n int) string { From 1972fe790b9887a9c156d61dee93585d2a24da44 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Wed, 21 Jan 2026 18:07:57 -0500 Subject: [PATCH 03/19] Implement a reusable internal/curl package --- internal/curl/exit_codes.go | 110 +++++++++++++++++++++++++ internal/curl/runner.go | 155 ++++++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 internal/curl/exit_codes.go create mode 100644 internal/curl/runner.go diff --git a/internal/curl/exit_codes.go b/internal/curl/exit_codes.go new file mode 100644 index 0000000..a5e53e9 --- /dev/null +++ b/internal/curl/exit_codes.go @@ -0,0 +1,110 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package curl + +// ExitCodeName returns the human readable name for a curl exit code. +func ExitCodeName(code int) string { + if name, ok := exitCodeNames[code]; ok { + return name + } + return "UNKNOWN_ERROR" +} + +var exitCodeNames = map[int]string{ + 1: "CURLE_UNSUPPORTED_PROTOCOL", + 2: "CURLE_FAILED_INIT", + 3: "CURLE_URL_MALFORMAT", + 4: "CURLE_NOT_BUILT_IN", + 5: "CURLE_COULDNT_RESOLVE_PROXY", + 6: "CURLE_COULDNT_RESOLVE_HOST", + 7: "CURLE_COULDNT_CONNECT", + 8: "CURLE_WEIRD_SERVER_REPLY", + 9: "CURLE_REMOTE_ACCESS_DENIED", + 11: "CURLE_FTP_WEIRD_PASV_REPLY", + 13: "CURLE_FTP_WEIRD_227_FORMAT", + 14: "CURLE_FTP_CANT_GET_HOST", + 15: "CURLE_FTP_CANT_RECONNECT", + 17: "CURLE_FTP_COULDNT_SET_TYPE", + 18: "CURLE_PARTIAL_FILE", + 19: "CURLE_FTP_COULDNT_RETR_FILE", + 21: "CURLE_QUOTE_ERROR", + 22: "CURLE_HTTP_RETURNED_ERROR", + 23: "CURLE_WRITE_ERROR", + 25: "CURLE_UPLOAD_FAILED", + 26: "CURLE_READ_ERROR", + 27: "CURLE_OUT_OF_MEMORY", + 28: "CURLE_OPERATION_TIMEDOUT", + 30: "CURLE_FTP_PORT_FAILED", + 31: "CURLE_FTP_COULDNT_USE_REST", + 33: "CURLE_RANGE_ERROR", + 34: "CURLE_HTTP_POST_ERROR", + 35: "CURLE_SSL_CONNECT_ERROR", + 36: "CURLE_BAD_DOWNLOAD_RESUME", + 37: "CURLE_FILE_COULDNT_READ_FILE", + 38: "CURLE_LDAP_CANNOT_BIND", + 39: "CURLE_LDAP_SEARCH_FAILED", + 41: "CURLE_FUNCTION_NOT_FOUND", + 42: "CURLE_ABORTED_BY_CALLBACK", + 43: "CURLE_BAD_FUNCTION_ARGUMENT", + 45: "CURLE_INTERFACE_FAILED", + 47: "CURLE_TOO_MANY_REDIRECTS", + 48: "CURLE_UNKNOWN_OPTION", + 49: "CURLE_TELNET_OPTION_SYNTAX", + 51: "CURLE_PEER_FAILED_VERIFICATION", + 52: "CURLE_GOT_NOTHING", + 53: "CURLE_SSL_ENGINE_NOTFOUND", + 54: "CURLE_SSL_ENGINE_SETFAILED", + 55: "CURLE_SEND_ERROR", + 56: "CURLE_RECV_ERROR", + 58: "CURLE_SSL_CERTPROBLEM", + 59: "CURLE_SSL_CIPHER", + 60: "CURLE_SSL_CACERT", + 61: "CURLE_BAD_CONTENT_ENCODING", + 62: "CURLE_LDAP_INVALID_URL", + 63: "CURLE_FILESIZE_EXCEEDED", + 64: "CURLE_USE_SSL_FAILED", + 65: "CURLE_SEND_FAIL_REWIND", + 66: "CURLE_SSL_ENGINE_INITFAILED", + 67: "CURLE_LOGIN_DENIED", + 68: "CURLE_TFTP_NOTFOUND", + 69: "CURLE_TFTP_PERM", + 70: "CURLE_REMOTE_DISK_FULL", + 71: "CURLE_TFTP_ILLEGAL", + 72: "CURLE_TFTP_UNKNOWNID", + 73: "CURLE_REMOTE_FILE_EXISTS", + 74: "CURLE_TFTP_NOSUCHUSER", + 75: "CURLE_CONV_FAILED", + 76: "CURLE_CONV_REQD", + 77: "CURLE_SSL_CACERT_BADFILE", + 78: "CURLE_REMOTE_FILE_NOT_FOUND", + 79: "CURLE_SSH", + 80: "CURLE_SSL_SHUTDOWN_FAILED", + 81: "CURLE_AGAIN", + 82: "CURLE_SSL_CRL_BADFILE", + 83: "CURLE_SSL_ISSUER_ERROR", + 84: "CURLE_FTP_PRET_FAILED", + 85: "CURLE_RTSP_CSEQ_ERROR", + 86: "CURLE_RTSP_SESSION_ERROR", + 87: "CURLE_FTP_BAD_FILE_LIST", + 88: "CURLE_CHUNK_FAILED", + 89: "CURLE_NO_CONNECTION_AVAILABLE", + 90: "CURLE_SSL_PINNEDPUBKEYNOTMATCH", + 91: "CURLE_SSL_INVALIDCERTSTATUS", + 92: "CURLE_HTTP2_STREAM", + 93: "CURLE_RECURSIVE_API_CALL", + 94: "CURLE_AUTH_ERROR", + 95: "CURLE_HTTP3", + 96: "CURLE_QUIC_CONNECT_ERROR", +} diff --git a/internal/curl/runner.go b/internal/curl/runner.go new file mode 100644 index 0000000..9747f53 --- /dev/null +++ b/internal/curl/runner.go @@ -0,0 +1,155 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package curl + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +// Runner handles the execution of a specific curl binary. +// It manages environment setup (e.g., LD_LIBRARY_PATH) required for custom builds. +type Runner struct { + curlPath string + libPath string +} + +// Args defines the execution parameters for a single curl run. +type Args struct { + // Proxy specifies the proxy URL to use. + // Corresponds to the "--proxy" flag. + Proxy string + + // ProxyHeaders specifies custom headers to send to the proxy. + // Each string should be in "Key: Value" format. + // Corresponds to the "--proxy-header" flag. + ProxyHeaders []string + + // ECH specifies the Encrypted ClientHello mode. + // Corresponds to the "--ech" flag. + // Use ECHGrease, ECHTrue, ECHFalse constants. + // If empty (ECHNone), the flag is omitted. + ECH ECHMode + + // Verbose enables verbose output. + // If true, adds "-v". + // If false, adds "-s" (silent mode) by default to keep output clean. + Verbose bool + + // Timeout sets the maximum time allowed for the transfer. + // Corresponds to the "--max-time" flag. + // If 0, no timeout is set. + Timeout time.Duration +} + +// ECHMode defines the available Encrypted ClientHello modes for curl. +type ECHMode string + +const ( + // ECHGrease enables ECH GREASE mode ("--ech grease"). + ECHGrease ECHMode = "grease" + // ECHTrue enables ECH ("--ech true"). + ECHTrue ECHMode = "true" + // ECHFalse disables ECH ("--ech false"). + ECHFalse ECHMode = "false" + // ECHNone indicates that the --ech flag should not be sent. + ECHNone ECHMode = "" +) + +// Result represents the raw outcome of a curl execution. +type Result struct { + // ExitCode is the exit status of the curl process. + // 0 indicates success. See ExitCodeName for error name mapping. + ExitCode int + + // Stdout contains the standard output of the curl command. + Stdout string + + // Stderr contains the standard error of the curl command. + // In verbose mode, this contains debug information and headers. + Stderr string +} + +// NewRunner creates a new Runner for the specified curl binary. +// It automatically detects the associated library path (bin/curl -> lib/) +// to ensure shared libraries are found. +func NewRunner(curlPath string) *Runner { + r := &Runner{curlPath: curlPath} + + binDir := filepath.Dir(curlPath) + libDir := filepath.Join(filepath.Dir(binDir), "lib") + if libStat, err := os.Stat(libDir); err == nil && libStat.IsDir() { + r.libPath = libDir + } + + return r +} + +// Run executes curl with the provided arguments and returns the result. +func (r *Runner) Run(url string, args Args) (*Result, error) { + var cmdArgs []string + + if args.Verbose { + cmdArgs = append(cmdArgs, "-v") + } else { + cmdArgs = append(cmdArgs, "-s") + } + + if args.Timeout > 0 { + cmdArgs = append(cmdArgs, "--max-time", strconv.FormatFloat(args.Timeout.Seconds(), 'f', -1, 64)) + } + + if args.Proxy != "" { + cmdArgs = append(cmdArgs, "--proxy", args.Proxy) + } + + for _, h := range args.ProxyHeaders { + cmdArgs = append(cmdArgs, "--proxy-header", h) + } + + if args.ECH != ECHNone { + cmdArgs = append(cmdArgs, "--ech", string(args.ECH)) + } + + cmdArgs = append(cmdArgs, url) + cmd := exec.Command(r.curlPath, cmdArgs...) + if r.libPath != "" { + cmd.Env = append(os.Environ(), "LD_LIBRARY_PATH="+r.libPath) + } + + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + result := &Result{} + err := cmd.Run() + result.Stdout = stdout.String() + result.Stderr = stderr.String() + + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + result.ExitCode = exitErr.ExitCode() + } else { + return result, fmt.Errorf("failed to execute curl: %w", err) + } + } + + return result, nil +} From ac756fbc7dbdc6f1df0ae2214a620969da23d84c Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 27 Jan 2026 16:22:12 -0500 Subject: [PATCH 04/19] implement performance stats collection via `curl -w` --- internal/curl/exit_codes.go | 3 ++ internal/curl/runner.go | 15 ++++++ internal/curl/stats.go | 101 ++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 internal/curl/stats.go diff --git a/internal/curl/exit_codes.go b/internal/curl/exit_codes.go index a5e53e9..e558665 100644 --- a/internal/curl/exit_codes.go +++ b/internal/curl/exit_codes.go @@ -16,6 +16,9 @@ package curl // ExitCodeName returns the human readable name for a curl exit code. func ExitCodeName(code int) string { + if code == 0 { + return "OK" + } if name, ok := exitCodeNames[code]; ok { return name } diff --git a/internal/curl/runner.go b/internal/curl/runner.go index 9747f53..8fb7df1 100644 --- a/internal/curl/runner.go +++ b/internal/curl/runner.go @@ -57,6 +57,10 @@ type Args struct { // Corresponds to the "--max-time" flag. // If 0, no timeout is set. Timeout time.Duration + + // MeasureStats enables capturing performance metrics using curl's -w flag. + // If true, Stats will be populated in the Result. + MeasureStats bool } // ECHMode defines the available Encrypted ClientHello modes for curl. @@ -85,6 +89,9 @@ type Result struct { // Stderr contains the standard error of the curl command. // In verbose mode, this contains debug information and headers. Stderr string + + // Stats contains performance metrics if MeasureStats was enabled. + Stats Stats } // NewRunner creates a new Runner for the specified curl binary. @@ -128,6 +135,10 @@ func (r *Runner) Run(url string, args Args) (*Result, error) { cmdArgs = append(cmdArgs, "--ech", string(args.ECH)) } + if args.MeasureStats { + cmdArgs = append(cmdArgs, "-w", statsFormat) + } + cmdArgs = append(cmdArgs, url) cmd := exec.Command(r.curlPath, cmdArgs...) if r.libPath != "" { @@ -143,6 +154,10 @@ func (r *Runner) Run(url string, args Args) (*Result, error) { result.Stdout = stdout.String() result.Stderr = stderr.String() + if args.MeasureStats { + result.Stats = parseStats(result.Stdout) + } + if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { result.ExitCode = exitErr.ExitCode() diff --git a/internal/curl/stats.go b/internal/curl/stats.go new file mode 100644 index 0000000..a450c55 --- /dev/null +++ b/internal/curl/stats.go @@ -0,0 +1,101 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package curl + +import ( + "strconv" + "strings" + "time" +) + +// Stats captures timing and status metrics from a curl execution. +type Stats struct { + // DNSLookupTimestamp is the cumulative time from the start until the name + // lookup is completed (time_namelookup). + DNSLookupTimestamp time.Duration + + // TCPConnectTimestamp is the cumulative time from the start until the TCP + // connection is completed (time_connect). + TCPConnectTimestamp time.Duration + + // TLSConnectTimestamp is the cumulative time from the start until the + // SSL/TLS handshake is completed (time_appconnect). + TLSConnectTimestamp time.Duration + + // ServerResponseTimestamp is the cumulative time from the start until the + // first byte is received (time_starttransfer). + ServerResponseTimestamp time.Duration + + // TotalTimeTimestamp is the total time from the start until the operation is + // fully completed (time_total). + TotalTimeTimestamp time.Duration + + // HTTPStatus is the HTTP response code (http_code). + HTTPStatus int +} + +const ( + // statsPrefix is the delimiter used to identify the statistics block in the output. + statsPrefix = "\n|||CURL_STATS|||\t" + + // statsFormat is the format string passed to curl's -w flag. + statsFormat = statsPrefix + + "dnslookup:%{time_namelookup}," + + "tcpconnect:%{time_connect}," + + "tlsconnect:%{time_appconnect}," + + "servertime:%{time_starttransfer}," + + "total:%{time_total}," + + "httpstatus:%{http_code}" +) + +// parseStats extracts Stats from the curl output by looking for the statsPrefix. +func parseStats(stdout string) Stats { + var s Stats + idx := strings.LastIndex(stdout, statsPrefix) + if idx == -1 { + return s + } + + raw := strings.TrimSpace(stdout[idx+len(statsPrefix):]) + for part := range strings.SplitSeq(raw, ",") { + kv := strings.Split(part, ":") + if len(kv) != 2 { + continue + } + + key, val := strings.TrimSpace(kv[0]), strings.TrimSpace(kv[1]) + switch key { + case "dnslookup": + s.DNSLookupTimestamp = parseDuration(val) + case "tcpconnect": + s.TCPConnectTimestamp = parseDuration(val) + case "tlsconnect": + s.TLSConnectTimestamp = parseDuration(val) + case "servertime": + s.ServerResponseTimestamp = parseDuration(val) + case "total": + s.TotalTimeTimestamp = parseDuration(val) + case "httpstatus": + s.HTTPStatus, _ = strconv.Atoi(val) + } + } + return s +} + +// parseDuration converts a seconds-based float string to time.Duration. +func parseDuration(s string) time.Duration { + f, _ := strconv.ParseFloat(s, 64) + return time.Duration(f * float64(time.Second)) +} From 55170b9328372d26d55fec46d1311c793cb11f31 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 27 Jan 2026 16:45:09 -0500 Subject: [PATCH 05/19] implement ECH testing with SOAX proxy --- soaxreport/main.go | 247 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 soaxreport/main.go diff --git a/soaxreport/main.go b/soaxreport/main.go new file mode 100644 index 0000000..2a786f2 --- /dev/null +++ b/soaxreport/main.go @@ -0,0 +1,247 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "encoding/csv" + "flag" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/Jigsaw-Code/ech-research/internal/curl" + "github.com/Jigsaw-Code/ech-research/internal/soax" + "github.com/Jigsaw-Code/ech-research/internal/workspace" +) + +type TestResult struct { + Domain string + Country string + ISP string + ASN string + ExitNodeIP string + ECHGrease bool + Error string + CurlExitCode int + CurlErrorName string + DNSLookup time.Duration + TCPConnection time.Duration + TLSHandshake time.Duration + ServerTime time.Duration + TotalTime time.Duration + HTTPStatus int +} + +func runSoaxTest(runner *curl.Runner, domain string, country string, isp string, proxyURL string, echGrease bool, maxTime time.Duration, verbose bool) TestResult { + result := TestResult{ + Domain: domain, + Country: country, + ISP: isp, + ECHGrease: echGrease, + } + + echMode := curl.ECHFalse + if echGrease { + echMode = curl.ECHGrease + } + + url := "https://" + domain + res, err := runner.Run(url, curl.Args{ + Proxy: proxyURL, + ProxyHeaders: []string{"Respond-With: ip,isp,asn"}, + ECH: echMode, + Timeout: maxTime, + Verbose: verbose, + MeasureStats: true, + }) + + result.CurlExitCode = res.ExitCode + result.CurlErrorName = curl.ExitCodeName(res.ExitCode) + result.HTTPStatus = res.Stats.HTTPStatus + result.DNSLookup = res.Stats.DNSLookupTimestamp + result.TCPConnection = res.Stats.TCPConnectTimestamp + result.TLSHandshake = res.Stats.TLSConnectTimestamp + result.ServerTime = res.Stats.ServerResponseTimestamp + result.TotalTime = res.Stats.TotalTimeTimestamp + + if err != nil { + result.Error = err.Error() + } + + // Parse metadata from Stderr (SOAX specific headers in CONNECT response) + for line := range strings.SplitSeq(res.Stderr, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "< Node-") { + continue + } + kv := strings.SplitN(strings.TrimPrefix(line, "< Node-"), ":", 2) + if len(kv) != 2 { + continue + } + key := strings.ToLower(strings.TrimSpace(kv[0])) + val := strings.TrimSpace(kv[1]) + switch key { + case "asn": + result.ASN = val + case "ip": + result.ExitNodeIP = val + case "isp": + result.ISP = val + } + } + + return result +} + +func loadCountries(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + var countries []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line != "" && !strings.HasPrefix(line, "#") { + countries = append(countries, line) + } + } + return countries, scanner.Err() +} + +func main() { + var ( + workspaceFlag = flag.String("workspace", "./workspace", "Directory to store intermediate files") + soaxConfigFlag = flag.String("soax", "", "Path to SOAX config JSON") + countriesFlag = flag.String("countries", "", "Path to file containing ISO country codes") + targetDomainFlag = flag.String("targetDomain", "www.google.com", "Target domain to test") + verboseFlag = flag.Bool("verbose", false, "Enable verbose logging") + maxTimeFlag = flag.Duration("maxTime", 30*time.Second, "Maximum time per curl request") + curlPathFlag = flag.String("curl", "", "Path to the ECH-enabled curl binary") + ) + flag.Parse() + + if *verboseFlag { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) + } else { + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))) + } + + // Set up workspace directory + workspaceDir := workspace.EnsureWorkspace(*workspaceFlag) + + // Determine curl binary path + curlPath := *curlPathFlag + if curlPath == "" { + curlPath = filepath.Join(workspaceDir, "output", "bin", "curl") + } + runner := curl.NewRunner(curlPath) + + // Load SOAX config + soaxConfigPath := *soaxConfigFlag + if soaxConfigPath == "" { + soaxConfigPath = filepath.Join(workspaceDir, "soax", "cred.json") + } + cfg, err := soax.LoadConfig(soaxConfigPath) + if err != nil { + slog.Error("Failed to load SOAX config", "path", soaxConfigPath, "error", err) + os.Exit(1) + } + client := soax.NewClient(cfg) + + // Load countries + if *countriesFlag == "" { + slog.Error("The --countries flag is required") + os.Exit(1) + } + countries, err := loadCountries(*countriesFlag) + if err != nil { + slog.Error("Failed to load countries list", "path", *countriesFlag, "error", err) + os.Exit(1) + } + + // Create output CSV file + outputFilename := filepath.Join(workspaceDir, fmt.Sprintf("soax-results-countries%d.csv", len(countries))) + outputFile, err := os.Create(outputFilename) + if err != nil { + slog.Error("Failed to create output CSV file", "path", outputFilename, "error", err) + os.Exit(1) + } + defer outputFile.Close() + + csvWriter := csv.NewWriter(outputFile) + defer csvWriter.Flush() + + header := []string{ + "domain", "country", "isp", "asn", "exit_node_ip", "ech_grease", "error", + "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", + "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", + } + if err := csvWriter.Write(header); err != nil { + slog.Error("Failed to write CSV header", "error", err) + os.Exit(1) + } + + domain := *targetDomainFlag + + for _, country := range countries { + slog.Info("Processing country", "country", country) + + isps, err := client.ListISPs(country) + if err != nil { + slog.Error("Failed to fetch ISPs", "country", country, "error", err) + continue + } + + for _, isp := range isps { + proxyURL := client.BuildProxyURL(country, isp, "") + + // Test ECH False + slog.Info("Testing ISP", "country", country, "isp", isp, "ech_grease", false) + resFalse := runSoaxTest(runner, domain, country, isp, proxyURL, false, *maxTimeFlag, *verboseFlag) + + // Test ECH Grease + slog.Info("Testing ISP", "country", country, "isp", isp, "ech_grease", true) + resGrease := runSoaxTest(runner, domain, country, isp, proxyURL, true, *maxTimeFlag, *verboseFlag) + + results := []TestResult{resFalse, resGrease} + for _, r := range results { + record := []string{ + r.Domain, r.Country, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, + strconv.Itoa(r.CurlExitCode), r.CurlErrorName, + strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), + strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), + strconv.FormatInt(r.TLSHandshake.Milliseconds(), 10), + strconv.FormatInt(r.ServerTime.Milliseconds(), 10), + strconv.FormatInt(r.TotalTime.Milliseconds(), 10), + strconv.Itoa(r.HTTPStatus), + } + if err := csvWriter.Write(record); err != nil { + slog.Error("Failed to write record to CSV", "error", err) + } + } + csvWriter.Flush() + } + } + + slog.Info("Done. Results saved to", "path", outputFilename) +} From 33397dd3c70186d0d212d3881cbf54bf26ea37e2 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Thu, 29 Jan 2026 00:03:38 -0500 Subject: [PATCH 06/19] Implement concurrent SOAX ECH testing --- soaxreport/README.md | 85 +++++++++++++++++++++++++++++ soaxreport/main.go | 126 ++++++++++++++++++++++++++++--------------- 2 files changed, 168 insertions(+), 43 deletions(-) create mode 100644 soaxreport/README.md diff --git a/soaxreport/README.md b/soaxreport/README.md new file mode 100644 index 0000000..a83b4f0 --- /dev/null +++ b/soaxreport/README.md @@ -0,0 +1,85 @@ +# SOAX ECH GREASE Report Generation + +This tool tests ECH GREASE compatibility by issuing requests via SOAX proxies. +It iterates through a list of countries and ISPs, running tests with and without +ECH GREASE to simulate diverse network vantage points. + +## Requirements + +You need to build the ECH-enabled `curl` and place it in the workspace directory. See [instructions](../curl/README.md). + +You also need a SOAX configuration file (`soax/cred.json` in the workspace) and a list of ISO country codes. + +### Configuration File Examples + +**SOAX Credentials (`soax/cred.json`)** + +The SOAX configuration file should be a JSON file with the following structure: + +```json +{ + "api_key": "YOUR_API_KEY", + "package_key": "YOUR_PACKAGE_KEY", + "package_id": "YOUR_PACKAGE_ID", + "proxy_host": "proxy.soax.com", + "proxy_port": 5000 +} +``` + +**Country List (`countries.txt`)** + +The countries file should contain a list of 2-letter ISO country codes, one per line. Lines starting with `#` are ignored. + +```text +US +GB +DE +# Add more countries as needed +JP +``` + +## Running + +To run the tool, use the `go run` command from the project root directory: + +```sh +go run ./soaxreport --countries workspace/countries.txt --targetDomain www.google.com +``` + +This will: + +1. Load the SOAX credentials (`./workspace/soax/cred.json` by default) and country list. +2. For each country, fetch the list of available ISPs. +3. For each ISP, issue requests to the target domain via a SOAX proxy, once with ECH GREASE and once without. +4. Save the results to `./workspace/soax-results--countries.csv`. + +### Parameters + +* `-workspace `: Directory to store intermediate files. Defaults to `./workspace`. +* `-soax `: Path to SOAX config JSON. Defaults to `./workspace/soax/cred.json`. +* `-countries `: Path to file containing ISO country codes (required). +* `-targetDomain `: Target domain to test. Defaults to `www.google.com`. +* `-parallelism `: Maximum number of parallel requests. Defaults to 10. +* `-verbose`: Enable verbose logging. +* `-maxTime `: Maximum time per curl request. Defaults to `30s`. +* `-curl `: Path to the ECH-enabled curl binary. Defaults to `./workspace/output/bin/curl`. + +### Output Format + +The tool generates a CSV file (`workspace/soax-results--countries.csv`) with the following columns: + +* `domain`: The domain that was tested. +* `country`: The country code of the proxy used. +* `isp`: The ISP name of the proxy used. +* `asn`: The ASN of the proxy exit node. +* `exit_node_ip`: The IP address of the proxy exit node. +* `ech_grease`: `true` if ECH GREASE was enabled for the request, `false` otherwise. +* `error`: Any error that occurred during the request. +* `curl_exit_code`: The exit code returned by the `curl` command. +* `curl_error_name`: The human-readable name corresponding to the `curl` exit code. +* `dns_lookup_ms`: The duration of the DNS lookup. +* `tcp_connection_ms`: The duration of the TCP connection. +* `tls_handshake_ms`: The duration of the TLS handshake. +* `server_time_ms`: The time from the end of the TLS handshake to the first byte of the response. +* `total_time_ms`: The total duration of the request. +* `http_status`: The HTTP status code of the response. diff --git a/soaxreport/main.go b/soaxreport/main.go index 2a786f2..e0e52a3 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -16,6 +16,7 @@ package main import ( "bufio" + "context" "encoding/csv" "flag" "fmt" @@ -24,11 +25,13 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "github.com/Jigsaw-Code/ech-research/internal/curl" "github.com/Jigsaw-Code/ech-research/internal/soax" "github.com/Jigsaw-Code/ech-research/internal/workspace" + "golang.org/x/sync/semaphore" ) type TestResult struct { @@ -49,7 +52,15 @@ type TestResult struct { HTTPStatus int } -func runSoaxTest(runner *curl.Runner, domain string, country string, isp string, proxyURL string, echGrease bool, maxTime time.Duration, verbose bool) TestResult { +func runSoaxTest( + runner *curl.Runner, + domain string, + country string, + isp string, + proxyURL string, + echGrease bool, + maxTime time.Duration, +) TestResult { result := TestResult{ Domain: domain, Country: country, @@ -68,7 +79,7 @@ func runSoaxTest(runner *curl.Runner, domain string, country string, isp string, ProxyHeaders: []string{"Respond-With: ip,isp,asn"}, ECH: echMode, Timeout: maxTime, - Verbose: verbose, + Verbose: true, // Required to capture response headers MeasureStats: true, }) @@ -103,7 +114,7 @@ func runSoaxTest(runner *curl.Runner, domain string, country string, isp string, case "ip": result.ExitNodeIP = val case "isp": - result.ISP = val + result.ISP += " (" + val + ")" } } @@ -137,6 +148,7 @@ func main() { verboseFlag = flag.Bool("verbose", false, "Enable verbose logging") maxTimeFlag = flag.Duration("maxTime", 30*time.Second, "Maximum time per curl request") curlPathFlag = flag.String("curl", "", "Path to the ECH-enabled curl binary") + parallelismFlag = flag.Int("parallelism", 10, "Maximum number of parallel requests") ) flag.Parse() @@ -180,7 +192,8 @@ func main() { } // Create output CSV file - outputFilename := filepath.Join(workspaceDir, fmt.Sprintf("soax-results-countries%d.csv", len(countries))) + sanitizedDomain := strings.ReplaceAll(*targetDomainFlag, ".", "_") + outputFilename := filepath.Join(workspaceDir, fmt.Sprintf("soax-results-%s-countries%d.csv", sanitizedDomain, len(countries))) outputFile, err := os.Create(outputFilename) if err != nil { slog.Error("Failed to create output CSV file", "path", outputFilename, "error", err) @@ -188,23 +201,46 @@ func main() { } defer outputFile.Close() - csvWriter := csv.NewWriter(outputFile) - defer csvWriter.Flush() + resultsCh := make(chan TestResult, 2*len(countries)*(*parallelismFlag)) - header := []string{ - "domain", "country", "isp", "asn", "exit_node_ip", "ech_grease", "error", - "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", - "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", - } - if err := csvWriter.Write(header); err != nil { - slog.Error("Failed to write CSV header", "error", err) - os.Exit(1) - } + var csvWg sync.WaitGroup + csvWg.Add(1) + go func() { + defer csvWg.Done() + csvWriter := csv.NewWriter(outputFile) + defer csvWriter.Flush() - domain := *targetDomainFlag + header := []string{ + "domain", "country", "isp", "asn", "exit_node_ip", "ech_grease", "error", + "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", + "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", + } + if err := csvWriter.Write(header); err != nil { + slog.Error("Failed to write CSV header", "error", err) + } + for r := range resultsCh { + record := []string{ + r.Domain, r.Country, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, + strconv.Itoa(r.CurlExitCode), r.CurlErrorName, + strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), + strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), + strconv.FormatInt(r.TLSHandshake.Milliseconds(), 10), + strconv.FormatInt(r.ServerTime.Milliseconds(), 10), + strconv.FormatInt(r.TotalTime.Milliseconds(), 10), + strconv.Itoa(r.HTTPStatus), + } + if err := csvWriter.Write(record); err != nil { + slog.Error("Failed to write record to CSV", "error", err) + } + } + }() + + domain := *targetDomainFlag + sem := semaphore.NewWeighted(int64(*parallelismFlag)) + var wg sync.WaitGroup for _, country := range countries { - slog.Info("Processing country", "country", country) + slog.Debug("Processing country", "country", country) isps, err := client.ListISPs(country) if err != nil { @@ -213,35 +249,39 @@ func main() { } for _, isp := range isps { - proxyURL := client.BuildProxyURL(country, isp, "") - - // Test ECH False - slog.Info("Testing ISP", "country", country, "isp", isp, "ech_grease", false) - resFalse := runSoaxTest(runner, domain, country, isp, proxyURL, false, *maxTimeFlag, *verboseFlag) - - // Test ECH Grease - slog.Info("Testing ISP", "country", country, "isp", isp, "ech_grease", true) - resGrease := runSoaxTest(runner, domain, country, isp, proxyURL, true, *maxTimeFlag, *verboseFlag) - - results := []TestResult{resFalse, resGrease} - for _, r := range results { - record := []string{ - r.Domain, r.Country, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, - strconv.Itoa(r.CurlExitCode), r.CurlErrorName, - strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), - strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), - strconv.FormatInt(r.TLSHandshake.Milliseconds(), 10), - strconv.FormatInt(r.ServerTime.Milliseconds(), 10), - strconv.FormatInt(r.TotalTime.Milliseconds(), 10), - strconv.Itoa(r.HTTPStatus), - } - if err := csvWriter.Write(record); err != nil { - slog.Error("Failed to write record to CSV", "error", err) - } + wg.Add(2) + + if err := sem.Acquire(context.Background(), 1); err != nil { + slog.Error("Failed to acquire semaphore", "error", err) + wg.Done() + } else { + go func(c, isp string) { + defer sem.Release(1) + defer wg.Done() + proxyURL := client.BuildProxyURL(c, isp, "") + slog.Info("Testing ISP", "country", c, "isp", isp, "ech_grease", false) + resultsCh <- runSoaxTest(runner, domain, c, isp, proxyURL, false, *maxTimeFlag) + }(country, isp) + } + + if err := sem.Acquire(context.Background(), 1); err != nil { + slog.Error("Failed to acquire semaphore", "error", err) + wg.Done() + } else { + go func(c, isp string) { + defer sem.Release(1) + defer wg.Done() + proxyURL := client.BuildProxyURL(c, isp, "") + slog.Info("Testing ISP", "country", c, "isp", isp, "ech_grease", true) + resultsCh <- runSoaxTest(runner, domain, c, isp, proxyURL, true, *maxTimeFlag) + }(country, isp) } - csvWriter.Flush() } } + wg.Wait() + close(resultsCh) + csvWg.Wait() + slog.Info("Done. Results saved to", "path", outputFilename) } From 07a674094ff294db82fbc4d47a4f241fb2abccc3 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Thu, 29 Jan 2026 01:16:38 -0500 Subject: [PATCH 07/19] Refactor to use CSV country list and include country names --- soaxreport/README.md | 19 ++++++----- soaxreport/main.go | 75 +++++++++++++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 34 deletions(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index a83b4f0..c1ccb89 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -28,16 +28,18 @@ The SOAX configuration file should be a JSON file with the following structure: **Country List (`countries.txt`)** -The countries file should contain a list of 2-letter ISO country codes, one per line. Lines starting with `#` are ignored. +The countries file should be a CSV file containing country names and their 2-letter ISO codes. Lines starting with `#` are ignored. -```text -US -GB -DE +```csv +"United States",US +"United Kingdom",GB +"Germany",DE # Add more countries as needed -JP +"Virgin Islands, U.S.",VI ``` +You can download a complete list of country codes from [here](https://raw.githubusercontent.com/datasets/country-list/master/data.csv). + ## Running To run the tool, use the `go run` command from the project root directory: @@ -57,7 +59,7 @@ This will: * `-workspace `: Directory to store intermediate files. Defaults to `./workspace`. * `-soax `: Path to SOAX config JSON. Defaults to `./workspace/soax/cred.json`. -* `-countries `: Path to file containing ISO country codes (required). +* `-countries `: Path to CSV file containing country names and ISO codes (required). * `-targetDomain `: Target domain to test. Defaults to `www.google.com`. * `-parallelism `: Maximum number of parallel requests. Defaults to 10. * `-verbose`: Enable verbose logging. @@ -69,7 +71,8 @@ This will: The tool generates a CSV file (`workspace/soax-results--countries.csv`) with the following columns: * `domain`: The domain that was tested. -* `country`: The country code of the proxy used. +* `country_code`: The 2-letter ISO country code. +* `country_name`: The full name of the country. * `isp`: The ISP name of the proxy used. * `asn`: The ASN of the proxy exit node. * `exit_node_ip`: The IP address of the proxy exit node. diff --git a/soaxreport/main.go b/soaxreport/main.go index e0e52a3..4a18faf 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -15,7 +15,6 @@ package main import ( - "bufio" "context" "encoding/csv" "flag" @@ -37,6 +36,7 @@ import ( type TestResult struct { Domain string Country string + CountryName string ISP string ASN string ExitNodeIP string @@ -56,16 +56,18 @@ func runSoaxTest( runner *curl.Runner, domain string, country string, + countryName string, isp string, proxyURL string, echGrease bool, maxTime time.Duration, ) TestResult { result := TestResult{ - Domain: domain, - Country: country, - ISP: isp, - ECHGrease: echGrease, + Domain: domain, + Country: country, + CountryName: countryName, + ISP: isp, + ECHGrease: echGrease, } echMode := curl.ECHFalse @@ -121,22 +123,43 @@ func runSoaxTest( return result } -func loadCountries(path string) ([]string, error) { +type Country struct { + Name string + Code string +} + +func loadCountries(path string) ([]Country, error) { f, err := os.Open(path) if err != nil { return nil, err } defer f.Close() - var countries []string - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line != "" && !strings.HasPrefix(line, "#") { - countries = append(countries, line) + var countries []Country + reader := csv.NewReader(f) + reader.Comment = '#' // Support skipping lines starting with # + reader.FieldsPerRecord = 2 + + records, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("failed to read countries CSV: %w", err) + } + + for i, record := range records { + name := strings.TrimSpace(record[0]) + code := strings.TrimSpace(record[1]) + + // Skip header row if present + if i == 0 && strings.EqualFold(name, "Name") && strings.EqualFold(code, "Code") { + continue } + + countries = append(countries, Country{ + Name: name, + Code: code, + }) } - return countries, scanner.Err() + return countries, nil } func main() { @@ -211,7 +234,7 @@ func main() { defer csvWriter.Flush() header := []string{ - "domain", "country", "isp", "asn", "exit_node_ip", "ech_grease", "error", + "domain", "country_code", "country_name", "isp", "asn", "exit_node_ip", "ech_grease", "error", "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", } @@ -221,7 +244,7 @@ func main() { for r := range resultsCh { record := []string{ - r.Domain, r.Country, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, + r.Domain, r.Country, r.CountryName, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, strconv.Itoa(r.CurlExitCode), r.CurlErrorName, strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), @@ -240,11 +263,11 @@ func main() { sem := semaphore.NewWeighted(int64(*parallelismFlag)) var wg sync.WaitGroup for _, country := range countries { - slog.Debug("Processing country", "country", country) + slog.Debug("Processing country", "name", country.Name, "code", country.Code) - isps, err := client.ListISPs(country) + isps, err := client.ListISPs(country.Code) if err != nil { - slog.Error("Failed to fetch ISPs", "country", country, "error", err) + slog.Error("Failed to fetch ISPs", "country", country.Code, "error", err) continue } @@ -255,12 +278,12 @@ func main() { slog.Error("Failed to acquire semaphore", "error", err) wg.Done() } else { - go func(c, isp string) { + go func(c Country, isp string) { defer sem.Release(1) defer wg.Done() - proxyURL := client.BuildProxyURL(c, isp, "") - slog.Info("Testing ISP", "country", c, "isp", isp, "ech_grease", false) - resultsCh <- runSoaxTest(runner, domain, c, isp, proxyURL, false, *maxTimeFlag) + proxyURL := client.BuildProxyURL(c.Code, isp, "") + slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", false) + resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, false, *maxTimeFlag) }(country, isp) } @@ -268,12 +291,12 @@ func main() { slog.Error("Failed to acquire semaphore", "error", err) wg.Done() } else { - go func(c, isp string) { + go func(c Country, isp string) { defer sem.Release(1) defer wg.Done() - proxyURL := client.BuildProxyURL(c, isp, "") - slog.Info("Testing ISP", "country", c, "isp", isp, "ech_grease", true) - resultsCh <- runSoaxTest(runner, domain, c, isp, proxyURL, true, *maxTimeFlag) + proxyURL := client.BuildProxyURL(c.Code, isp, "") + slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", true) + resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, true, *maxTimeFlag) }(country, isp) } } From d7f5a50b533cac587b55cbc16cc07c25bca40d5c Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Thu, 29 Jan 2026 01:31:16 -0500 Subject: [PATCH 08/19] Ensure same exit node for one ISP thru sticky session. --- soaxreport/main.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/soaxreport/main.go b/soaxreport/main.go index 4a18faf..afdb070 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -260,6 +260,7 @@ func main() { }() domain := *targetDomainFlag + runSessionID := time.Now().Format("0102150405") sem := semaphore.NewWeighted(int64(*parallelismFlag)) var wg sync.WaitGroup for _, country := range countries { @@ -271,33 +272,34 @@ func main() { continue } - for _, isp := range isps { + for i, isp := range isps { wg.Add(2) + sessionID := fmt.Sprintf("%s%s%d", runSessionID, country.Code, i) if err := sem.Acquire(context.Background(), 1); err != nil { slog.Error("Failed to acquire semaphore", "error", err) wg.Done() } else { - go func(c Country, isp string) { + go func(c Country, isp, sid string) { defer sem.Release(1) defer wg.Done() - proxyURL := client.BuildProxyURL(c.Code, isp, "") - slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", false) + proxyURL := client.BuildProxyURL(c.Code, isp, sid) + slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", false, "session", sid) resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, false, *maxTimeFlag) - }(country, isp) + }(country, isp, sessionID) } if err := sem.Acquire(context.Background(), 1); err != nil { slog.Error("Failed to acquire semaphore", "error", err) wg.Done() } else { - go func(c Country, isp string) { + go func(c Country, isp, sid string) { defer sem.Release(1) defer wg.Done() - proxyURL := client.BuildProxyURL(c.Code, isp, "") - slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", true) + proxyURL := client.BuildProxyURL(c.Code, isp, sid) + slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", true, "session", sid) resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, true, *maxTimeFlag) - }(country, isp) + }(country, isp, sessionID) } } } From e6fa052dcbdf0ff51bf3c96c0f534a8d81e2459b Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Mon, 2 Feb 2026 16:34:28 -0500 Subject: [PATCH 09/19] Implement atomic progress tracking and simplify test func --- soaxreport/README.md | 4 ++-- soaxreport/main.go | 44 ++++++++++++++++++++------------------------ 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index c1ccb89..71ad5f3 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -26,7 +26,7 @@ The SOAX configuration file should be a JSON file with the following structure: } ``` -**Country List (`countries.txt`)** +**Country List (`countries.csv`)** The countries file should be a CSV file containing country names and their 2-letter ISO codes. Lines starting with `#` are ignored. @@ -45,7 +45,7 @@ You can download a complete list of country codes from [here](https://raw.github To run the tool, use the `go run` command from the project root directory: ```sh -go run ./soaxreport --countries workspace/countries.txt --targetDomain www.google.com +go run ./soaxreport --countries workspace/countries.csv --targetDomain www.google.com ``` This will: diff --git a/soaxreport/main.go b/soaxreport/main.go index afdb070..805665a 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/Jigsaw-Code/ech-research/internal/curl" @@ -171,7 +172,7 @@ func main() { verboseFlag = flag.Bool("verbose", false, "Enable verbose logging") maxTimeFlag = flag.Duration("maxTime", 30*time.Second, "Maximum time per curl request") curlPathFlag = flag.String("curl", "", "Path to the ECH-enabled curl binary") - parallelismFlag = flag.Int("parallelism", 10, "Maximum number of parallel requests") + parallelismFlag = flag.Int("parallelism", 16, "Maximum number of parallel requests") ) flag.Parse() @@ -263,6 +264,8 @@ func main() { runSessionID := time.Now().Format("0102150405") sem := semaphore.NewWeighted(int64(*parallelismFlag)) var wg sync.WaitGroup + var total, finished atomic.Int32 + for _, country := range countries { slog.Debug("Processing country", "name", country.Name, "code", country.Code) @@ -272,35 +275,28 @@ func main() { continue } + total.Add(int32(len(isps) * 2)) for i, isp := range isps { wg.Add(2) sessionID := fmt.Sprintf("%s%s%d", runSessionID, country.Code, i) - if err := sem.Acquire(context.Background(), 1); err != nil { - slog.Error("Failed to acquire semaphore", "error", err) - wg.Done() - } else { - go func(c Country, isp, sid string) { - defer sem.Release(1) - defer wg.Done() - proxyURL := client.BuildProxyURL(c.Code, isp, sid) - slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", false, "session", sid) - resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, false, *maxTimeFlag) - }(country, isp, sessionID) + startTest := func(c Country, isp, sid string, ech bool) { + defer wg.Done() + if err := sem.Acquire(context.Background(), 1); err != nil { + slog.Error("Failed to acquire semaphore", "error", err) + return + } + defer sem.Release(1) + + proxyURL := client.BuildProxyURL(c.Code, isp, sid) + slog.Debug("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", ech, "session", sid) + resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, ech, *maxTimeFlag) + progress := fmt.Sprintf("%d/%d", finished.Add(1), total.Load()) + slog.Info("Finished", "country", c.Code, "isp", isp, "progress", progress) } - if err := sem.Acquire(context.Background(), 1); err != nil { - slog.Error("Failed to acquire semaphore", "error", err) - wg.Done() - } else { - go func(c Country, isp, sid string) { - defer sem.Release(1) - defer wg.Done() - proxyURL := client.BuildProxyURL(c.Code, isp, sid) - slog.Info("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", true, "session", sid) - resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, true, *maxTimeFlag) - }(country, isp, sessionID) - } + go startTest(country, isp, sessionID, false) + go startTest(country, isp, sessionID, true) } } From e978f1e4aac3c4fa82acc6ebe0e8bd797db75f2d Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Mon, 2 Feb 2026 17:32:20 -0500 Subject: [PATCH 10/19] Separate input ISP and exit node ISP header into different columns. --- soaxreport/README.md | 1 + soaxreport/main.go | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index 71ad5f3..76cd4af 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -76,6 +76,7 @@ The tool generates a CSV file (`workspace/soax-results--countries.csv * `isp`: The ISP name of the proxy used. * `asn`: The ASN of the proxy exit node. * `exit_node_ip`: The IP address of the proxy exit node. +* `exit_node_isp`: The ISP name reported by the proxy exit node (from headers). * `ech_grease`: `true` if ECH GREASE was enabled for the request, `false` otherwise. * `error`: Any error that occurred during the request. * `curl_exit_code`: The exit code returned by the `curl` command. diff --git a/soaxreport/main.go b/soaxreport/main.go index 805665a..cc08279 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -41,6 +41,7 @@ type TestResult struct { ISP string ASN string ExitNodeIP string + ExitNodeISP string ECHGrease bool Error string CurlExitCode int @@ -117,7 +118,7 @@ func runSoaxTest( case "ip": result.ExitNodeIP = val case "isp": - result.ISP += " (" + val + ")" + result.ExitNodeISP = val } } @@ -235,7 +236,7 @@ func main() { defer csvWriter.Flush() header := []string{ - "domain", "country_code", "country_name", "isp", "asn", "exit_node_ip", "ech_grease", "error", + "domain", "country_code", "country_name", "isp", "asn", "exit_node_ip", "exit_node_isp", "ech_grease", "error", "curl_exit_code", "curl_error_name", "dns_lookup_ms", "tcp_connection_ms", "tls_handshake_ms", "server_time_ms", "total_time_ms", "http_status", } @@ -245,7 +246,7 @@ func main() { for r := range resultsCh { record := []string{ - r.Domain, r.Country, r.CountryName, r.ISP, r.ASN, r.ExitNodeIP, strconv.FormatBool(r.ECHGrease), r.Error, + r.Domain, r.Country, r.CountryName, r.ISP, r.ASN, r.ExitNodeIP, r.ExitNodeISP, strconv.FormatBool(r.ECHGrease), r.Error, strconv.Itoa(r.CurlExitCode), r.CurlErrorName, strconv.FormatInt(r.DNSLookup.Milliseconds(), 10), strconv.FormatInt(r.TCPConnection.Milliseconds(), 10), From f0557b7c5fc2fda68bd1c29abd40e1cbba61fe11 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Wed, 4 Feb 2026 16:50:06 -0500 Subject: [PATCH 11/19] Update parallelism default value in README for soaxreport --- soaxreport/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index 76cd4af..ee54c91 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -61,7 +61,7 @@ This will: * `-soax `: Path to SOAX config JSON. Defaults to `./workspace/soax/cred.json`. * `-countries `: Path to CSV file containing country names and ISO codes (required). * `-targetDomain `: Target domain to test. Defaults to `www.google.com`. -* `-parallelism `: Maximum number of parallel requests. Defaults to 10. +* `-parallelism `: Maximum number of parallel requests. Defaults to `16`. * `-verbose`: Enable verbose logging. * `-maxTime `: Maximum time per curl request. Defaults to `30s`. * `-curl `: Path to the ECH-enabled curl binary. Defaults to `./workspace/output/bin/curl`. From 1ade342b4b1bfc70e2709f1f1ce405834b49ec86 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 10 Mar 2026 22:43:38 +0800 Subject: [PATCH 12/19] refactor(soax): rename BuildProxyURL to BuildWebProxyURL --- internal/soax/soax.go | 4 ++-- soaxreport/main.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/soax/soax.go b/internal/soax/soax.go index 264ff81..d48fdd0 100644 --- a/internal/soax/soax.go +++ b/internal/soax/soax.go @@ -91,9 +91,9 @@ func (c *Client) ListISPs(countryISO string) ([]string, error) { return isps, nil } -// BuildProxyURL constructs an authenticated HTTPS proxy URL for a specific country and ISP. +// BuildWebProxyURL constructs an authenticated HTTPS proxy URL for a specific country and ISP. // An optional sessionID can be provided for sticky sessions; if empty, a random one is generated. -func (c *Client) BuildProxyURL(countryISO, ispName, sessionID string) string { +func (c *Client) BuildWebProxyURL(countryISO, ispName, sessionID string) string { if sessionID == "" { sessionID = generateRandomString(10) } diff --git a/soaxreport/main.go b/soaxreport/main.go index cc08279..b6f3c42 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -289,7 +289,7 @@ func main() { } defer sem.Release(1) - proxyURL := client.BuildProxyURL(c.Code, isp, sid) + proxyURL := client.BuildWebProxyURL(c.Code, isp, sid) slog.Debug("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", ech, "session", sid) resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, ech, *maxTimeFlag) progress := fmt.Sprintf("%d/%d", finished.Add(1), total.Load()) From 679ca18aae4a14351ed4a8e3a0ffed07f6fb8e35 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 10 Mar 2026 22:51:23 +0800 Subject: [PATCH 13/19] feat(soax): use outline-sdk for ISP retrieval --- go.mod | 11 +++++++++ go.sum | 56 +++++++++++++++++++++++++++++++++++++++++++ internal/soax/soax.go | 41 ++++++++++++++++++++----------- 3 files changed, 94 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 07f7a56..f2d3099 100644 --- a/go.mod +++ b/go.mod @@ -4,13 +4,24 @@ go 1.24.8 require ( github.com/miekg/dns v1.1.70 + golang.getoutline.org/sdk/x v0.1.0 golang.org/x/sync v0.19.0 ) require ( + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect + github.com/onsi/ginkgo/v2 v2.12.0 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.48.1 // indirect + go.uber.org/mock v0.4.0 // indirect + golang.getoutline.org/sdk v0.0.21 // indirect + golang.org/x/crypto v0.46.0 // indirect + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.31.0 // indirect golang.org/x/net v0.48.0 // indirect golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/tools v0.40.0 // indirect ) diff --git a/go.sum b/go.sum index d224094..da637d3 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,70 @@ +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/miekg/dns v1.1.70 h1:DZ4u2AV35VJxdD9Fo9fIWm119BsQL5cZU1cQ9s0LkqA= github.com/miekg/dns v1.1.70/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= +github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= +github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA= +github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= +github.com/shadowsocks/go-shadowsocks2 v0.1.5 h1:PDSQv9y2S85Fl7VBeOMF9StzeXZyK1HakRm86CUbr28= +github.com/shadowsocks/go-shadowsocks2 v0.1.5/go.mod h1:AGGpIoek4HRno4xzyFiAtLHkOpcoznZEkAccaI/rplM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/things-go/go-socks5 v0.0.5 h1:qvKaGcBkfDrUL33SchHN93srAmYGzb4CxSM2DPYufe8= +github.com/things-go/go-socks5 v0.0.5/go.mod h1:mtzInf8v5xmsBpHZVbIw2YQYhc4K0jRwzfsH64Uh0IQ= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +golang.getoutline.org/sdk v0.0.21 h1:zgtenz5DMbnIPOsuAOHNiWdrri81fHyBxhSfRi6Dk8s= +golang.getoutline.org/sdk v0.0.21/go.mod h1:raUAs4PYbEaT/cLTK6PviiKSh7gjEj7JJczFFFr41zc= +golang.getoutline.org/sdk/x v0.1.0 h1:8ykaCEC8Eoi3h/2MdGW7uaMAt2BWFCRhrSvuJ0Y/IU0= +golang.getoutline.org/sdk/x v0.1.0/go.mod h1:Vw7FWpLbYifHFYbbo0mXOCkhR14d1ADwjiF7uBQKyzM= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/soax/soax.go b/internal/soax/soax.go index d48fdd0..4af8bd7 100644 --- a/internal/soax/soax.go +++ b/internal/soax/soax.go @@ -15,14 +15,15 @@ package soax import ( + "context" "encoding/json" "fmt" - "io" "math/rand" - "net/http" "net/url" "os" "strings" + + "golang.getoutline.org/sdk/x/soax" ) // Config holds the credentials and endpoint configuration for the SOAX service. @@ -37,6 +38,7 @@ type Config struct { // Client provides methods to interact with the SOAX API and generate proxy configurations. type Client struct { cfg *Config + sdk *soax.Client } // LoadConfig reads the SOAX configuration from a JSON file. @@ -64,29 +66,40 @@ func LoadConfig(path string) (*Config, error) { // NewClient creates a new SOAX Client with the given configuration. func NewClient(cfg *Config) *Client { - return &Client{cfg: cfg} + return &Client{ + cfg: cfg, + sdk: &soax.Client{ + APIKey: cfg.APIKey, + PackageKey: cfg.PackageKey, + }, + } } // ListISPs retrieves a list of available ISP operators for the specified country code. // countryISO should be a 2-letter ISO country code (e.g., "US"). func (c *Client) ListISPs(countryISO string) ([]string, error) { - url := fmt.Sprintf("https://api.soax.com/api/get-country-operators?api_key=%s&package_key=%s&country_iso=%s", - c.cfg.APIKey, c.cfg.PackageKey, strings.ToLower(countryISO)) + ctx := context.Background() + ispMap := make(map[string]bool) - resp, err := http.Get(url) - if err != nil { - return nil, fmt.Errorf("failed to fetch ISPs: %w", err) + if mIsps, err := c.sdk.GetMobileISPs(ctx, countryISO, "", ""); err == nil { + for _, isp := range mIsps { + ispMap[isp] = true + } + } + + if rIsps, err := c.sdk.GetResidentialISPs(ctx, countryISO, "", ""); err == nil { + for _, isp := range rIsps { + ispMap[isp] = true + } } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("API error: status %s, body: %s", resp.Status, string(body)) + if len(ispMap) == 0 { + return nil, fmt.Errorf("no ISPs found for country %s", countryISO) } var isps []string - if err := json.NewDecoder(resp.Body).Decode(&isps); err != nil { - return nil, fmt.Errorf("failed to decode ISP list: %w", err) + for isp := range ispMap { + isps = append(isps, isp) } return isps, nil } From 73a30231877461e913eaed851680110a51889be0 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 10 Mar 2026 23:01:14 +0800 Subject: [PATCH 14/19] fix(soax): use url.URL for proper escaping in BuildWebProxyURL --- internal/soax/soax.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/internal/soax/soax.go b/internal/soax/soax.go index 4af8bd7..3201f20 100644 --- a/internal/soax/soax.go +++ b/internal/soax/soax.go @@ -111,14 +111,25 @@ func (c *Client) BuildWebProxyURL(countryISO, ispName, sessionID string) string sessionID = generateRandomString(10) } - ispName = url.QueryEscape(strings.ToLower(ispName)) - countryISO = strings.ToLower(countryISO) + params := []string{"package", c.cfg.PackageID} + if countryISO != "" { + params = append(params, "country", strings.ToLower(countryISO)) + } + if ispName != "" { + params = append(params, "isp", strings.ToLower(ispName)) + } + if sessionID != "" { + params = append(params, "sessionid", sessionID) + } + params = append(params, "sessionlength", "300") - proxyUser := fmt.Sprintf("package-%s-country-%s-isp-%s-sessionid-%s-sessionlength-300", - c.cfg.PackageID, countryISO, ispName, sessionID) + u := &url.URL{ + Scheme: "https", + User: url.UserPassword(strings.Join(params, "-"), c.cfg.PackageKey), + Host: fmt.Sprintf("%s:%d", c.cfg.ProxyHost, c.cfg.ProxyPort), + } - return fmt.Sprintf("https://%s:%s@%s:%d", - proxyUser, c.cfg.PackageKey, c.cfg.ProxyHost, c.cfg.ProxyPort) + return u.String() } func generateRandomString(n int) string { From 7b218ce9b9d7a6a2a2edbc413a5ccdc2e7dcceab Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 10 Mar 2026 23:07:10 +0800 Subject: [PATCH 15/19] chore: add deduplication TODO for exit codes --- greasereport/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/greasereport/main.go b/greasereport/main.go index 70a6156..565cd8f 100644 --- a/greasereport/main.go +++ b/greasereport/main.go @@ -49,6 +49,7 @@ type TestResult struct { HTTPStatus int } +// TODO: Deduplicate this with the unified ECH testing package (internal/echtest). var curlExitCodeNames = map[int]string{ 1: "CURLE_UNSUPPORTED_PROTOCOL", 2: "CURLE_FAILED_INIT", From 31f28b85f885976b516fd22b1a5c13fa4321e0bd Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 10 Mar 2026 23:22:17 +0800 Subject: [PATCH 16/19] refactor(echtest): extract runTest logic and remove translation layer --- internal/curl/exit_codes.go | 113 ----------------- internal/curl/runner.go | 170 ------------------------- internal/curl/stats.go | 101 --------------- internal/echtest/run.go | 242 ++++++++++++++++++++++++++++++++++++ soaxreport/main.go | 65 +++------- 5 files changed, 256 insertions(+), 435 deletions(-) delete mode 100644 internal/curl/exit_codes.go delete mode 100644 internal/curl/runner.go delete mode 100644 internal/curl/stats.go create mode 100644 internal/echtest/run.go diff --git a/internal/curl/exit_codes.go b/internal/curl/exit_codes.go deleted file mode 100644 index e558665..0000000 --- a/internal/curl/exit_codes.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package curl - -// ExitCodeName returns the human readable name for a curl exit code. -func ExitCodeName(code int) string { - if code == 0 { - return "OK" - } - if name, ok := exitCodeNames[code]; ok { - return name - } - return "UNKNOWN_ERROR" -} - -var exitCodeNames = map[int]string{ - 1: "CURLE_UNSUPPORTED_PROTOCOL", - 2: "CURLE_FAILED_INIT", - 3: "CURLE_URL_MALFORMAT", - 4: "CURLE_NOT_BUILT_IN", - 5: "CURLE_COULDNT_RESOLVE_PROXY", - 6: "CURLE_COULDNT_RESOLVE_HOST", - 7: "CURLE_COULDNT_CONNECT", - 8: "CURLE_WEIRD_SERVER_REPLY", - 9: "CURLE_REMOTE_ACCESS_DENIED", - 11: "CURLE_FTP_WEIRD_PASV_REPLY", - 13: "CURLE_FTP_WEIRD_227_FORMAT", - 14: "CURLE_FTP_CANT_GET_HOST", - 15: "CURLE_FTP_CANT_RECONNECT", - 17: "CURLE_FTP_COULDNT_SET_TYPE", - 18: "CURLE_PARTIAL_FILE", - 19: "CURLE_FTP_COULDNT_RETR_FILE", - 21: "CURLE_QUOTE_ERROR", - 22: "CURLE_HTTP_RETURNED_ERROR", - 23: "CURLE_WRITE_ERROR", - 25: "CURLE_UPLOAD_FAILED", - 26: "CURLE_READ_ERROR", - 27: "CURLE_OUT_OF_MEMORY", - 28: "CURLE_OPERATION_TIMEDOUT", - 30: "CURLE_FTP_PORT_FAILED", - 31: "CURLE_FTP_COULDNT_USE_REST", - 33: "CURLE_RANGE_ERROR", - 34: "CURLE_HTTP_POST_ERROR", - 35: "CURLE_SSL_CONNECT_ERROR", - 36: "CURLE_BAD_DOWNLOAD_RESUME", - 37: "CURLE_FILE_COULDNT_READ_FILE", - 38: "CURLE_LDAP_CANNOT_BIND", - 39: "CURLE_LDAP_SEARCH_FAILED", - 41: "CURLE_FUNCTION_NOT_FOUND", - 42: "CURLE_ABORTED_BY_CALLBACK", - 43: "CURLE_BAD_FUNCTION_ARGUMENT", - 45: "CURLE_INTERFACE_FAILED", - 47: "CURLE_TOO_MANY_REDIRECTS", - 48: "CURLE_UNKNOWN_OPTION", - 49: "CURLE_TELNET_OPTION_SYNTAX", - 51: "CURLE_PEER_FAILED_VERIFICATION", - 52: "CURLE_GOT_NOTHING", - 53: "CURLE_SSL_ENGINE_NOTFOUND", - 54: "CURLE_SSL_ENGINE_SETFAILED", - 55: "CURLE_SEND_ERROR", - 56: "CURLE_RECV_ERROR", - 58: "CURLE_SSL_CERTPROBLEM", - 59: "CURLE_SSL_CIPHER", - 60: "CURLE_SSL_CACERT", - 61: "CURLE_BAD_CONTENT_ENCODING", - 62: "CURLE_LDAP_INVALID_URL", - 63: "CURLE_FILESIZE_EXCEEDED", - 64: "CURLE_USE_SSL_FAILED", - 65: "CURLE_SEND_FAIL_REWIND", - 66: "CURLE_SSL_ENGINE_INITFAILED", - 67: "CURLE_LOGIN_DENIED", - 68: "CURLE_TFTP_NOTFOUND", - 69: "CURLE_TFTP_PERM", - 70: "CURLE_REMOTE_DISK_FULL", - 71: "CURLE_TFTP_ILLEGAL", - 72: "CURLE_TFTP_UNKNOWNID", - 73: "CURLE_REMOTE_FILE_EXISTS", - 74: "CURLE_TFTP_NOSUCHUSER", - 75: "CURLE_CONV_FAILED", - 76: "CURLE_CONV_REQD", - 77: "CURLE_SSL_CACERT_BADFILE", - 78: "CURLE_REMOTE_FILE_NOT_FOUND", - 79: "CURLE_SSH", - 80: "CURLE_SSL_SHUTDOWN_FAILED", - 81: "CURLE_AGAIN", - 82: "CURLE_SSL_CRL_BADFILE", - 83: "CURLE_SSL_ISSUER_ERROR", - 84: "CURLE_FTP_PRET_FAILED", - 85: "CURLE_RTSP_CSEQ_ERROR", - 86: "CURLE_RTSP_SESSION_ERROR", - 87: "CURLE_FTP_BAD_FILE_LIST", - 88: "CURLE_CHUNK_FAILED", - 89: "CURLE_NO_CONNECTION_AVAILABLE", - 90: "CURLE_SSL_PINNEDPUBKEYNOTMATCH", - 91: "CURLE_SSL_INVALIDCERTSTATUS", - 92: "CURLE_HTTP2_STREAM", - 93: "CURLE_RECURSIVE_API_CALL", - 94: "CURLE_AUTH_ERROR", - 95: "CURLE_HTTP3", - 96: "CURLE_QUIC_CONNECT_ERROR", -} diff --git a/internal/curl/runner.go b/internal/curl/runner.go deleted file mode 100644 index 8fb7df1..0000000 --- a/internal/curl/runner.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package curl - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "time" -) - -// Runner handles the execution of a specific curl binary. -// It manages environment setup (e.g., LD_LIBRARY_PATH) required for custom builds. -type Runner struct { - curlPath string - libPath string -} - -// Args defines the execution parameters for a single curl run. -type Args struct { - // Proxy specifies the proxy URL to use. - // Corresponds to the "--proxy" flag. - Proxy string - - // ProxyHeaders specifies custom headers to send to the proxy. - // Each string should be in "Key: Value" format. - // Corresponds to the "--proxy-header" flag. - ProxyHeaders []string - - // ECH specifies the Encrypted ClientHello mode. - // Corresponds to the "--ech" flag. - // Use ECHGrease, ECHTrue, ECHFalse constants. - // If empty (ECHNone), the flag is omitted. - ECH ECHMode - - // Verbose enables verbose output. - // If true, adds "-v". - // If false, adds "-s" (silent mode) by default to keep output clean. - Verbose bool - - // Timeout sets the maximum time allowed for the transfer. - // Corresponds to the "--max-time" flag. - // If 0, no timeout is set. - Timeout time.Duration - - // MeasureStats enables capturing performance metrics using curl's -w flag. - // If true, Stats will be populated in the Result. - MeasureStats bool -} - -// ECHMode defines the available Encrypted ClientHello modes for curl. -type ECHMode string - -const ( - // ECHGrease enables ECH GREASE mode ("--ech grease"). - ECHGrease ECHMode = "grease" - // ECHTrue enables ECH ("--ech true"). - ECHTrue ECHMode = "true" - // ECHFalse disables ECH ("--ech false"). - ECHFalse ECHMode = "false" - // ECHNone indicates that the --ech flag should not be sent. - ECHNone ECHMode = "" -) - -// Result represents the raw outcome of a curl execution. -type Result struct { - // ExitCode is the exit status of the curl process. - // 0 indicates success. See ExitCodeName for error name mapping. - ExitCode int - - // Stdout contains the standard output of the curl command. - Stdout string - - // Stderr contains the standard error of the curl command. - // In verbose mode, this contains debug information and headers. - Stderr string - - // Stats contains performance metrics if MeasureStats was enabled. - Stats Stats -} - -// NewRunner creates a new Runner for the specified curl binary. -// It automatically detects the associated library path (bin/curl -> lib/) -// to ensure shared libraries are found. -func NewRunner(curlPath string) *Runner { - r := &Runner{curlPath: curlPath} - - binDir := filepath.Dir(curlPath) - libDir := filepath.Join(filepath.Dir(binDir), "lib") - if libStat, err := os.Stat(libDir); err == nil && libStat.IsDir() { - r.libPath = libDir - } - - return r -} - -// Run executes curl with the provided arguments and returns the result. -func (r *Runner) Run(url string, args Args) (*Result, error) { - var cmdArgs []string - - if args.Verbose { - cmdArgs = append(cmdArgs, "-v") - } else { - cmdArgs = append(cmdArgs, "-s") - } - - if args.Timeout > 0 { - cmdArgs = append(cmdArgs, "--max-time", strconv.FormatFloat(args.Timeout.Seconds(), 'f', -1, 64)) - } - - if args.Proxy != "" { - cmdArgs = append(cmdArgs, "--proxy", args.Proxy) - } - - for _, h := range args.ProxyHeaders { - cmdArgs = append(cmdArgs, "--proxy-header", h) - } - - if args.ECH != ECHNone { - cmdArgs = append(cmdArgs, "--ech", string(args.ECH)) - } - - if args.MeasureStats { - cmdArgs = append(cmdArgs, "-w", statsFormat) - } - - cmdArgs = append(cmdArgs, url) - cmd := exec.Command(r.curlPath, cmdArgs...) - if r.libPath != "" { - cmd.Env = append(os.Environ(), "LD_LIBRARY_PATH="+r.libPath) - } - - var stdout, stderr strings.Builder - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - result := &Result{} - err := cmd.Run() - result.Stdout = stdout.String() - result.Stderr = stderr.String() - - if args.MeasureStats { - result.Stats = parseStats(result.Stdout) - } - - if err != nil { - if exitErr, ok := err.(*exec.ExitError); ok { - result.ExitCode = exitErr.ExitCode() - } else { - return result, fmt.Errorf("failed to execute curl: %w", err) - } - } - - return result, nil -} diff --git a/internal/curl/stats.go b/internal/curl/stats.go deleted file mode 100644 index a450c55..0000000 --- a/internal/curl/stats.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package curl - -import ( - "strconv" - "strings" - "time" -) - -// Stats captures timing and status metrics from a curl execution. -type Stats struct { - // DNSLookupTimestamp is the cumulative time from the start until the name - // lookup is completed (time_namelookup). - DNSLookupTimestamp time.Duration - - // TCPConnectTimestamp is the cumulative time from the start until the TCP - // connection is completed (time_connect). - TCPConnectTimestamp time.Duration - - // TLSConnectTimestamp is the cumulative time from the start until the - // SSL/TLS handshake is completed (time_appconnect). - TLSConnectTimestamp time.Duration - - // ServerResponseTimestamp is the cumulative time from the start until the - // first byte is received (time_starttransfer). - ServerResponseTimestamp time.Duration - - // TotalTimeTimestamp is the total time from the start until the operation is - // fully completed (time_total). - TotalTimeTimestamp time.Duration - - // HTTPStatus is the HTTP response code (http_code). - HTTPStatus int -} - -const ( - // statsPrefix is the delimiter used to identify the statistics block in the output. - statsPrefix = "\n|||CURL_STATS|||\t" - - // statsFormat is the format string passed to curl's -w flag. - statsFormat = statsPrefix + - "dnslookup:%{time_namelookup}," + - "tcpconnect:%{time_connect}," + - "tlsconnect:%{time_appconnect}," + - "servertime:%{time_starttransfer}," + - "total:%{time_total}," + - "httpstatus:%{http_code}" -) - -// parseStats extracts Stats from the curl output by looking for the statsPrefix. -func parseStats(stdout string) Stats { - var s Stats - idx := strings.LastIndex(stdout, statsPrefix) - if idx == -1 { - return s - } - - raw := strings.TrimSpace(stdout[idx+len(statsPrefix):]) - for part := range strings.SplitSeq(raw, ",") { - kv := strings.Split(part, ":") - if len(kv) != 2 { - continue - } - - key, val := strings.TrimSpace(kv[0]), strings.TrimSpace(kv[1]) - switch key { - case "dnslookup": - s.DNSLookupTimestamp = parseDuration(val) - case "tcpconnect": - s.TCPConnectTimestamp = parseDuration(val) - case "tlsconnect": - s.TLSConnectTimestamp = parseDuration(val) - case "servertime": - s.ServerResponseTimestamp = parseDuration(val) - case "total": - s.TotalTimeTimestamp = parseDuration(val) - case "httpstatus": - s.HTTPStatus, _ = strconv.Atoi(val) - } - } - return s -} - -// parseDuration converts a seconds-based float string to time.Duration. -func parseDuration(s string) time.Duration { - f, _ := strconv.ParseFloat(s, 64) - return time.Duration(f * float64(time.Second)) -} diff --git a/internal/echtest/run.go b/internal/echtest/run.go new file mode 100644 index 0000000..67d1df8 --- /dev/null +++ b/internal/echtest/run.go @@ -0,0 +1,242 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package echtest + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" +) + +type TestResult struct { + Domain string + ECHGrease bool + Error string + CurlExitCode int + CurlErrorName string + DNSLookup time.Duration + TCPConnection time.Duration + TLSHandshake time.Duration + ServerTime time.Duration + TotalTime time.Duration + HTTPStatus int + Stderr string +} + +// curlExitCodeNames maps curl exit codes to their CURL_* string representations. +var curlExitCodeNames = map[int]string{ + 1: "CURLE_UNSUPPORTED_PROTOCOL", + 2: "CURLE_FAILED_INIT", + 3: "CURLE_URL_MALFORMAT", + 4: "CURLE_NOT_BUILT_IN", + 5: "CURLE_COULDNT_RESOLVE_PROXY", + 6: "CURLE_COULDNT_RESOLVE_HOST", + 7: "CURLE_COULDNT_CONNECT", + 8: "CURLE_WEIRD_SERVER_REPLY", + 9: "CURLE_REMOTE_ACCESS_DENIED", + 11: "CURLE_FTP_WEIRD_PASV_REPLY", + 13: "CURLE_FTP_WEIRD_227_FORMAT", + 14: "CURLE_FTP_CANT_GET_HOST", + 15: "CURLE_FTP_CANT_RECONNECT", + 17: "CURLE_FTP_COULDNT_SET_TYPE", + 18: "CURLE_PARTIAL_FILE", + 19: "CURLE_FTP_COULDNT_RETR_FILE", + 21: "CURLE_QUOTE_ERROR", + 22: "CURLE_HTTP_RETURNED_ERROR", + 23: "CURLE_WRITE_ERROR", + 25: "CURLE_UPLOAD_FAILED", + 26: "CURLE_READ_ERROR", + 27: "CURLE_OUT_OF_MEMORY", + 28: "CURLE_OPERATION_TIMEDOUT", + 30: "CURLE_FTP_PORT_FAILED", + 31: "CURLE_FTP_COULDNT_USE_REST", + 33: "CURLE_RANGE_ERROR", + 34: "CURLE_HTTP_POST_ERROR", + 35: "CURLE_SSL_CONNECT_ERROR", + 36: "CURLE_BAD_DOWNLOAD_RESUME", + 37: "CURLE_FILE_COULDNT_READ_FILE", + 38: "CURLE_LDAP_CANNOT_BIND", + 39: "CURLE_LDAP_SEARCH_FAILED", + 41: "CURLE_FUNCTION_NOT_FOUND", + 42: "CURLE_ABORTED_BY_CALLBACK", + 43: "CURLE_BAD_FUNCTION_ARGUMENT", + 45: "CURLE_INTERFACE_FAILED", + 47: "CURLE_TOO_MANY_REDIRECTS", + 48: "CURLE_UNKNOWN_OPTION", + 49: "CURLE_TELNET_OPTION_SYNTAX", + 51: "CURLE_PEER_FAILED_VERIFICATION", + 52: "CURLE_GOT_NOTHING", + 53: "CURLE_SSL_ENGINE_NOTFOUND", + 54: "CURLE_SSL_ENGINE_SETFAILED", + 55: "CURLE_SEND_ERROR", + 56: "CURLE_RECV_ERROR", + 58: "CURLE_SSL_CERTPROBLEM", + 59: "CURLE_SSL_CIPHER", + 60: "CURLE_SSL_CACERT", + 61: "CURLE_BAD_CONTENT_ENCODING", + 62: "CURLE_LDAP_INVALID_URL", + 63: "CURLE_FILESIZE_EXCEEDED", + 64: "CURLE_USE_SSL_FAILED", + 65: "CURLE_SEND_FAIL_REWIND", + 66: "CURLE_SSL_ENGINE_INITFAILED", + 67: "CURLE_LOGIN_DENIED", + 68: "CURLE_TFTP_NOTFOUND", + 69: "CURLE_TFTP_PERM", + 70: "CURLE_REMOTE_DISK_FULL", + 71: "CURLE_TFTP_ILLEGAL", + 72: "CURLE_TFTP_UNKNOWNID", + 73: "CURLE_REMOTE_FILE_EXISTS", + 74: "CURLE_TFTP_NOSUCHUSER", + 75: "CURLE_CONV_FAILED", + 76: "CURLE_CONV_REQD", + 77: "CURLE_SSL_CACERT_BADFILE", + 78: "CURLE_REMOTE_FILE_NOT_FOUND", + 79: "CURLE_SSH", + 80: "CURLE_SSL_SHUTDOWN_FAILED", + 81: "CURLE_AGAIN", + 82: "CURLE_SSL_CRL_BADFILE", + 83: "CURLE_SSL_ISSUER_ERROR", + 84: "CURLE_FTP_PRET_FAILED", + 85: "CURLE_RTSP_CSEQ_ERROR", + 86: "CURLE_RTSP_SESSION_ERROR", + 87: "CURLE_FTP_BAD_FILE_LIST", + 88: "CURLE_CHUNK_FAILED", + 89: "CURLE_NO_CONNECTION_AVAILABLE", + 90: "CURLE_SSL_PINNEDPUBKEYNOTMATCH", + 91: "CURLE_SSL_INVALIDCERTSTATUS", + 92: "CURLE_HTTP2_STREAM", + 93: "CURLE_RECURSIVE_API_CALL", + 94: "CURLE_AUTH_ERROR", + 95: "CURLE_HTTP3", + 96: "CURLE_QUIC_CONNECT_ERROR", +} + +// Run executes a curl command against the specified domain. +func Run( + curlPath string, + domain string, + echGrease bool, + maxTime time.Duration, + proxyURL string, + proxyHeaders []string, +) TestResult { + result := TestResult{ + Domain: domain, + ECHGrease: echGrease, + } + + targetURL := "https://" + domain + + args := []string{ + "-w", + "dnslookup:%{time_namelookup},tcpconnect:%{time_connect},tlsconnect:%{time_appconnect},servertime:%{time_starttransfer},total:%{time_total},httpstatus:%{http_code}", + "--head", + "--max-time", + strconv.FormatFloat(maxTime.Seconds(), 'f', -1, 64), + } + + // Handle proxy options + if proxyURL != "" { + args = append(args, "--proxy", proxyURL) + for _, h := range proxyHeaders { + args = append(args, "--proxy-header", h) + } + // If using a proxy with headers, we usually need verbose mode to see the proxy response. + // If proxy headers are provided, we assume the caller wants to read them from stderr. + if len(proxyHeaders) > 0 { + args = append(args, "-v") + } else { + args = append(args, "-s") + } + } else { + args = append(args, "-s") + } + + if echGrease { + args = append(args, "--ech", "grease") + } else { + args = append(args, "--ech", "false") + } + args = append(args, targetURL) + + cmd := exec.Command(curlPath, args...) + + // Setup environment for custom curl (matching internal/curl/runner.go) + binDir := filepath.Dir(curlPath) + libDir := filepath.Join(filepath.Dir(binDir), "lib") + if libStat, err := os.Stat(libDir); err == nil && libStat.IsDir() { + cmd.Env = append(os.Environ(), "LD_LIBRARY_PATH="+libDir) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + result.Stderr = stderr.String() // Always capture stderr for caller + + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + result.CurlExitCode = exitError.ExitCode() + result.CurlErrorName = curlExitCodeNames[result.CurlExitCode] + } else { + result.Error = fmt.Sprintf("failed to execute curl: %v", err) + return result + } + } else { + // Even if err is nil, there might be curl-level errors recorded in stderr + // that the caller might be interested in, though standard execution succeeded. + } + + // parse the stdout stats + parts := strings.SplitSeq(stdout.String(), ",") + for part := range parts { + kv := strings.Split(part, ":") + if len(kv) != 2 { + continue + } + key := kv[0] + value := kv[1] + + switch key { + case "dnslookup": + f, _ := strconv.ParseFloat(value, 64) + result.DNSLookup = time.Duration(f * float64(time.Second)) + case "tcpconnect": + f, _ := strconv.ParseFloat(value, 64) + result.TCPConnection = time.Duration(f * float64(time.Second)) + case "tlsconnect": + f, _ := strconv.ParseFloat(value, 64) + result.TLSHandshake = time.Duration(f * float64(time.Second)) + case "servertime": + f, _ := strconv.ParseFloat(value, 64) + result.ServerTime = time.Duration(f * float64(time.Second)) + case "total": + f, _ := strconv.ParseFloat(value, 64) + result.TotalTime = time.Duration(f * float64(time.Second)) + case "httpstatus": + i, _ := strconv.Atoi(value) + result.HTTPStatus = i + } + } + + return result +} diff --git a/soaxreport/main.go b/soaxreport/main.go index b6f3c42..bb6f026 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -28,34 +28,24 @@ import ( "sync/atomic" "time" - "github.com/Jigsaw-Code/ech-research/internal/curl" + "github.com/Jigsaw-Code/ech-research/internal/echtest" "github.com/Jigsaw-Code/ech-research/internal/soax" "github.com/Jigsaw-Code/ech-research/internal/workspace" "golang.org/x/sync/semaphore" ) type TestResult struct { - Domain string - Country string - CountryName string - ISP string - ASN string - ExitNodeIP string - ExitNodeISP string - ECHGrease bool - Error string - CurlExitCode int - CurlErrorName string - DNSLookup time.Duration - TCPConnection time.Duration - TLSHandshake time.Duration - ServerTime time.Duration - TotalTime time.Duration - HTTPStatus int + echtest.TestResult + Country string + CountryName string + ISP string + ASN string + ExitNodeIP string + ExitNodeISP string } func runSoaxTest( - runner *curl.Runner, + curlPath string, domain string, country string, countryName string, @@ -64,40 +54,14 @@ func runSoaxTest( echGrease bool, maxTime time.Duration, ) TestResult { + headers := []string{"Respond-With: ip,isp,asn"} + res := echtest.Run(curlPath, domain, echGrease, maxTime, proxyURL, headers) + result := TestResult{ - Domain: domain, + TestResult: res, Country: country, CountryName: countryName, ISP: isp, - ECHGrease: echGrease, - } - - echMode := curl.ECHFalse - if echGrease { - echMode = curl.ECHGrease - } - - url := "https://" + domain - res, err := runner.Run(url, curl.Args{ - Proxy: proxyURL, - ProxyHeaders: []string{"Respond-With: ip,isp,asn"}, - ECH: echMode, - Timeout: maxTime, - Verbose: true, // Required to capture response headers - MeasureStats: true, - }) - - result.CurlExitCode = res.ExitCode - result.CurlErrorName = curl.ExitCodeName(res.ExitCode) - result.HTTPStatus = res.Stats.HTTPStatus - result.DNSLookup = res.Stats.DNSLookupTimestamp - result.TCPConnection = res.Stats.TCPConnectTimestamp - result.TLSHandshake = res.Stats.TLSConnectTimestamp - result.ServerTime = res.Stats.ServerResponseTimestamp - result.TotalTime = res.Stats.TotalTimeTimestamp - - if err != nil { - result.Error = err.Error() } // Parse metadata from Stderr (SOAX specific headers in CONNECT response) @@ -191,7 +155,6 @@ func main() { if curlPath == "" { curlPath = filepath.Join(workspaceDir, "output", "bin", "curl") } - runner := curl.NewRunner(curlPath) // Load SOAX config soaxConfigPath := *soaxConfigFlag @@ -291,7 +254,7 @@ func main() { proxyURL := client.BuildWebProxyURL(c.Code, isp, sid) slog.Debug("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", ech, "session", sid) - resultsCh <- runSoaxTest(runner, domain, c.Code, c.Name, isp, proxyURL, ech, *maxTimeFlag) + resultsCh <- runSoaxTest(curlPath, domain, c.Code, c.Name, isp, proxyURL, ech, *maxTimeFlag) progress := fmt.Sprintf("%d/%d", finished.Add(1), total.Load()) slog.Info("Finished", "country", c.Code, "isp", isp, "progress", progress) } From 8c67c39f0bcdf3553519f8fbb9b613c0bc2a1c7e Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 10 Mar 2026 23:29:13 +0800 Subject: [PATCH 17/19] feat(soaxreport): set default path for countries list --- soaxreport/README.md | 6 +++--- soaxreport/main.go | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index ee54c91..fc7f6aa 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -45,12 +45,12 @@ You can download a complete list of country codes from [here](https://raw.github To run the tool, use the `go run` command from the project root directory: ```sh -go run ./soaxreport --countries workspace/countries.csv --targetDomain www.google.com +go run ./soaxreport --targetDomain www.google.com ``` This will: -1. Load the SOAX credentials (`./workspace/soax/cred.json` by default) and country list. +1. Load the SOAX credentials (`./workspace/soax/cred.json` by default) and country list (`./workspace/countries.csv` by default). 2. For each country, fetch the list of available ISPs. 3. For each ISP, issue requests to the target domain via a SOAX proxy, once with ECH GREASE and once without. 4. Save the results to `./workspace/soax-results--countries.csv`. @@ -59,7 +59,7 @@ This will: * `-workspace `: Directory to store intermediate files. Defaults to `./workspace`. * `-soax `: Path to SOAX config JSON. Defaults to `./workspace/soax/cred.json`. -* `-countries `: Path to CSV file containing country names and ISO codes (required). +* `-countries `: Path to CSV file containing country names and ISO codes. Defaults to `./workspace/countries.csv`. * `-targetDomain `: Target domain to test. Defaults to `www.google.com`. * `-parallelism `: Maximum number of parallel requests. Defaults to `16`. * `-verbose`: Enable verbose logging. diff --git a/soaxreport/main.go b/soaxreport/main.go index bb6f026..b2b3a87 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -169,13 +169,13 @@ func main() { client := soax.NewClient(cfg) // Load countries - if *countriesFlag == "" { - slog.Error("The --countries flag is required") - os.Exit(1) + countriesPath := *countriesFlag + if countriesPath == "" { + countriesPath = filepath.Join(workspaceDir, "countries.csv") } - countries, err := loadCountries(*countriesFlag) + countries, err := loadCountries(countriesPath) if err != nil { - slog.Error("Failed to load countries list", "path", *countriesFlag, "error", err) + slog.Error("Failed to load countries list", "path", countriesPath, "error", err) os.Exit(1) } From 80e9d6c76aa7e158ecaeca78798ceb719251de79 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Tue, 10 Mar 2026 23:34:49 +0800 Subject: [PATCH 18/19] feat(soaxreport): save discovered ISPs to audit JSON file --- soaxreport/README.md | 7 ++++++- soaxreport/main.go | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/soaxreport/README.md b/soaxreport/README.md index fc7f6aa..8a43291 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -68,7 +68,12 @@ This will: ### Output Format -The tool generates a CSV file (`workspace/soax-results--countries.csv`) with the following columns: +The tool generates two output files in the workspace directory: + +1. **Results CSV** (`workspace/soax-results--countries.csv`): Contains the detailed test results for each request. +2. **ISP Audit Log** (`workspace/soax-isps-audit.json`): A JSON file mapping each country code to the list of ISPs discovered and used during the test. This is useful for auditing coverage. + +The CSV file contains the following columns: * `domain`: The domain that was tested. * `country_code`: The 2-letter ISO country code. diff --git a/soaxreport/main.go b/soaxreport/main.go index b2b3a87..a612df0 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -17,6 +17,7 @@ package main import ( "context" "encoding/csv" + "encoding/json" "flag" "fmt" "log/slog" @@ -230,6 +231,9 @@ func main() { var wg sync.WaitGroup var total, finished atomic.Int32 + // Audit map to store discovered ISPs per country + ispAuditMap := make(map[string][]string) + for _, country := range countries { slog.Debug("Processing country", "name", country.Name, "code", country.Code) @@ -239,6 +243,8 @@ func main() { continue } + ispAuditMap[country.Code] = isps + total.Add(int32(len(isps) * 2)) for i, isp := range isps { wg.Add(2) @@ -268,5 +274,18 @@ func main() { close(resultsCh) csvWg.Wait() + // Write the ISP audit log to JSON + auditFilename := filepath.Join(workspaceDir, "soax-isps-audit.json") + auditData, err := json.MarshalIndent(ispAuditMap, "", " ") + if err == nil { + if err := os.WriteFile(auditFilename, auditData, 0644); err != nil { + slog.Error("Failed to write ISP audit log", "error", err) + } else { + slog.Info("ISP audit log saved", "path", auditFilename) + } + } else { + slog.Error("Failed to marshal ISP audit log", "error", err) + } + slog.Info("Done. Results saved to", "path", outputFilename) } From 70443e288aefb070f1eea53313e111802b062c42 Mon Sep 17 00:00:00 2001 From: Junyi Yi Date: Wed, 11 Mar 2026 00:02:06 +0800 Subject: [PATCH 19/19] refactor(soax): remove Client wrapper and use environment variables for config --- internal/soax/soax.go | 91 +++++++++++++++++++++---------------------- soaxreport/README.md | 28 +++++++------ soaxreport/main.go | 22 +++++------ 3 files changed, 68 insertions(+), 73 deletions(-) diff --git a/internal/soax/soax.go b/internal/soax/soax.go index 3201f20..dd24910 100644 --- a/internal/soax/soax.go +++ b/internal/soax/soax.go @@ -16,11 +16,10 @@ package soax import ( "context" - "encoding/json" "fmt" "math/rand" "net/url" - "os" + "strconv" "strings" "golang.getoutline.org/sdk/x/soax" @@ -28,66 +27,64 @@ import ( // Config holds the credentials and endpoint configuration for the SOAX service. type Config struct { - APIKey string `json:"api_key"` - PackageKey string `json:"package_key"` - PackageID string `json:"package_id"` - ProxyHost string `json:"proxy_host"` - ProxyPort int `json:"proxy_port"` + APIKey string + PackageKey string + PackageID string + ProxyHost string + ProxyPort int } -// Client provides methods to interact with the SOAX API and generate proxy configurations. -type Client struct { - cfg *Config - sdk *soax.Client -} - -// LoadConfig reads the SOAX configuration from a JSON file. -// If ProxyHost or ProxyPort are missing in the config, default values are used. -func LoadConfig(path string) (*Config, error) { - f, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("failed to open config file: %w", err) +// NewConfig creates a new Config, validating required fields and setting defaults. +func NewConfig(apiKey, packageKey, packageID, proxyHost, proxyPortStr string) (*Config, error) { + if apiKey == "" { + return nil, fmt.Errorf("API key is required") } - defer f.Close() - - var cfg Config - if err := json.NewDecoder(f).Decode(&cfg); err != nil { - return nil, fmt.Errorf("failed to decode config json: %w", err) + if packageKey == "" { + return nil, fmt.Errorf("package key is required") } - if cfg.ProxyHost == "" { - cfg.ProxyHost = "proxy.soax.com" + if packageID == "" { + return nil, fmt.Errorf("package ID is required") } - if cfg.ProxyPort == 0 { - cfg.ProxyPort = 5000 + if proxyHost == "" { + proxyHost = "proxy.soax.com" } - return &cfg, nil -} - -// NewClient creates a new SOAX Client with the given configuration. -func NewClient(cfg *Config) *Client { - return &Client{ - cfg: cfg, - sdk: &soax.Client{ - APIKey: cfg.APIKey, - PackageKey: cfg.PackageKey, - }, + proxyPort := 5000 + if proxyPortStr != "" { + p, err := strconv.Atoi(proxyPortStr) + if err != nil { + return nil, fmt.Errorf("invalid proxy port %q: %v", proxyPortStr, err) + } + proxyPort = p } + + return &Config{ + APIKey: apiKey, + PackageKey: packageKey, + PackageID: packageID, + ProxyHost: proxyHost, + ProxyPort: proxyPort, + }, nil } // ListISPs retrieves a list of available ISP operators for the specified country code. -// countryISO should be a 2-letter ISO country code (e.g., "US"). -func (c *Client) ListISPs(countryISO string) ([]string, error) { +// It combines both mobile and residential ISPs using the provided SDK client. +func ListISPs(cfg *Config, countryISO string) ([]string, error) { + sdkClient := &soax.Client{ + APIKey: cfg.APIKey, + PackageKey: cfg.PackageKey, + } + ctx := context.Background() ispMap := make(map[string]bool) - if mIsps, err := c.sdk.GetMobileISPs(ctx, countryISO, "", ""); err == nil { + if mIsps, err := sdkClient.GetMobileISPs(ctx, countryISO, "", ""); err == nil { for _, isp := range mIsps { ispMap[isp] = true } } - if rIsps, err := c.sdk.GetResidentialISPs(ctx, countryISO, "", ""); err == nil { + if rIsps, err := sdkClient.GetResidentialISPs(ctx, countryISO, "", ""); err == nil { for _, isp := range rIsps { ispMap[isp] = true } @@ -106,12 +103,12 @@ func (c *Client) ListISPs(countryISO string) ([]string, error) { // BuildWebProxyURL constructs an authenticated HTTPS proxy URL for a specific country and ISP. // An optional sessionID can be provided for sticky sessions; if empty, a random one is generated. -func (c *Client) BuildWebProxyURL(countryISO, ispName, sessionID string) string { +func BuildWebProxyURL(cfg *Config, countryISO, ispName, sessionID string) string { if sessionID == "" { sessionID = generateRandomString(10) } - params := []string{"package", c.cfg.PackageID} + params := []string{"package", cfg.PackageID} if countryISO != "" { params = append(params, "country", strings.ToLower(countryISO)) } @@ -125,8 +122,8 @@ func (c *Client) BuildWebProxyURL(countryISO, ispName, sessionID string) string u := &url.URL{ Scheme: "https", - User: url.UserPassword(strings.Join(params, "-"), c.cfg.PackageKey), - Host: fmt.Sprintf("%s:%d", c.cfg.ProxyHost, c.cfg.ProxyPort), + User: url.UserPassword(strings.Join(params, "-"), cfg.PackageKey), + Host: fmt.Sprintf("%s:%d", cfg.ProxyHost, cfg.ProxyPort), } return u.String() diff --git a/soaxreport/README.md b/soaxreport/README.md index 8a43291..0338d7b 100644 --- a/soaxreport/README.md +++ b/soaxreport/README.md @@ -8,22 +8,21 @@ ECH GREASE to simulate diverse network vantage points. You need to build the ECH-enabled `curl` and place it in the workspace directory. See [instructions](../curl/README.md). -You also need a SOAX configuration file (`soax/cred.json` in the workspace) and a list of ISO country codes. +You also need to set the SOAX credentials as environment variables and provide a list of ISO country codes. -### Configuration File Examples +### Configuration -**SOAX Credentials (`soax/cred.json`)** +**SOAX Credentials (Environment Variables)** -The SOAX configuration file should be a JSON file with the following structure: +Set the following environment variables with your SOAX API details: -```json -{ - "api_key": "YOUR_API_KEY", - "package_key": "YOUR_PACKAGE_KEY", - "package_id": "YOUR_PACKAGE_ID", - "proxy_host": "proxy.soax.com", - "proxy_port": 5000 -} +```bash +export SOAX_API_KEY="YOUR_API_KEY" +export SOAX_PACKAGE_KEY="YOUR_PACKAGE_KEY" +export SOAX_PACKAGE_ID="YOUR_PACKAGE_ID" +# Optional overrides: +# export SOAX_PROXY_HOST="proxy.soax.com" +# export SOAX_PROXY_PORT="5000" ``` **Country List (`countries.csv`)** @@ -42,7 +41,7 @@ You can download a complete list of country codes from [here](https://raw.github ## Running -To run the tool, use the `go run` command from the project root directory: +To run the tool, ensure your environment variables are set, then use the `go run` command from the project root directory: ```sh go run ./soaxreport --targetDomain www.google.com @@ -50,7 +49,7 @@ go run ./soaxreport --targetDomain www.google.com This will: -1. Load the SOAX credentials (`./workspace/soax/cred.json` by default) and country list (`./workspace/countries.csv` by default). +1. Load the SOAX credentials from the environment and the country list (`./workspace/countries.csv` by default). 2. For each country, fetch the list of available ISPs. 3. For each ISP, issue requests to the target domain via a SOAX proxy, once with ECH GREASE and once without. 4. Save the results to `./workspace/soax-results--countries.csv`. @@ -58,7 +57,6 @@ This will: ### Parameters * `-workspace `: Directory to store intermediate files. Defaults to `./workspace`. -* `-soax `: Path to SOAX config JSON. Defaults to `./workspace/soax/cred.json`. * `-countries `: Path to CSV file containing country names and ISO codes. Defaults to `./workspace/countries.csv`. * `-targetDomain `: Target domain to test. Defaults to `www.google.com`. * `-parallelism `: Maximum number of parallel requests. Defaults to `16`. diff --git a/soaxreport/main.go b/soaxreport/main.go index a612df0..fb41cdc 100644 --- a/soaxreport/main.go +++ b/soaxreport/main.go @@ -132,7 +132,6 @@ func loadCountries(path string) ([]Country, error) { func main() { var ( workspaceFlag = flag.String("workspace", "./workspace", "Directory to store intermediate files") - soaxConfigFlag = flag.String("soax", "", "Path to SOAX config JSON") countriesFlag = flag.String("countries", "", "Path to file containing ISO country codes") targetDomainFlag = flag.String("targetDomain", "www.google.com", "Target domain to test") verboseFlag = flag.Bool("verbose", false, "Enable verbose logging") @@ -157,17 +156,18 @@ func main() { curlPath = filepath.Join(workspaceDir, "output", "bin", "curl") } - // Load SOAX config - soaxConfigPath := *soaxConfigFlag - if soaxConfigPath == "" { - soaxConfigPath = filepath.Join(workspaceDir, "soax", "cred.json") - } - cfg, err := soax.LoadConfig(soaxConfigPath) + // Load SOAX config from environment variables + cfg, err := soax.NewConfig( + os.Getenv("SOAX_API_KEY"), + os.Getenv("SOAX_PACKAGE_KEY"), + os.Getenv("SOAX_PACKAGE_ID"), + os.Getenv("SOAX_PROXY_HOST"), + os.Getenv("SOAX_PROXY_PORT"), + ) if err != nil { - slog.Error("Failed to load SOAX config", "path", soaxConfigPath, "error", err) + slog.Error("Failed to initialize SOAX config from environment", "error", err) os.Exit(1) } - client := soax.NewClient(cfg) // Load countries countriesPath := *countriesFlag @@ -237,7 +237,7 @@ func main() { for _, country := range countries { slog.Debug("Processing country", "name", country.Name, "code", country.Code) - isps, err := client.ListISPs(country.Code) + isps, err := soax.ListISPs(cfg, country.Code) if err != nil { slog.Error("Failed to fetch ISPs", "country", country.Code, "error", err) continue @@ -258,7 +258,7 @@ func main() { } defer sem.Release(1) - proxyURL := client.BuildWebProxyURL(c.Code, isp, sid) + proxyURL := soax.BuildWebProxyURL(cfg, c.Code, isp, sid) slog.Debug("Testing ISP", "country", c.Code, "isp", isp, "ech_grease", ech, "session", sid) resultsCh <- runSoaxTest(curlPath, domain, c.Code, c.Name, isp, proxyURL, ech, *maxTimeFlag) progress := fmt.Sprintf("%d/%d", finished.Add(1), total.Load())