diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f069bf3..92e1748 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -61,7 +61,7 @@ jobs: run: go build ./... verify-linux-amd64: - name: Verify Linux amd64 + name: Verify (ubuntu-latest) runs-on: ubuntu-latest steps: - name: Checkout @@ -107,3 +107,20 @@ jobs: run: go mod tidy -diff - name: Check whitespace run: git diff --check + + live-parser-regression: + name: Live Parser Regression + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + - name: Parse authorized surveys on all providers + env: + SURVEYCORE_LIVE_PARSE: "1" + run: go test -count=1 -timeout=3m ./CI/live_tests -run '^TestLiveProviderParse$' -v diff --git a/CI/live_tests/provider_submission_test.go b/CI/live_tests/provider_submission_test.go index dfabb90..8d8ab29 100644 --- a/CI/live_tests/provider_submission_test.go +++ b/CI/live_tests/provider_submission_test.go @@ -20,6 +20,42 @@ var ( sensitiveFieldValue = regexp.MustCompile(`(?i)(access[_-]?token|refresh[_-]?token|api[_-]?key|password|passwd)(\s*[=:]\s*|\s+)[^\s,;}&]+`) ) +func TestLiveProviderParse(t *testing.T) { + if os.Getenv("SURVEYCORE_LIVE_PARSE") != "1" { + t.Skip("set SURVEYCORE_LIVE_PARSE=1 to enable live parser checks") + } + for _, survey := range []struct { + provider string + url string + parseError string + }{ + {model.ProviderWJX, "https://v.wjx.cn/vm/ei3sVrE.aspx", ""}, + // This published fixture includes rating types the parser explicitly rejects. + {model.ProviderQQ, "https://wj.qq.com/s2/26070328/fa89/", "腾讯问卷当前版本暂不支持量表、矩阵量表题"}, + {model.ProviderCredamo, "https://www.credamo.com/s/A73QR3ano", ""}, + } { + t.Run(survey.provider, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + if survey.parseError != "" { + _, err := surveycore.New().Parse(ctx, survey.url) + if !errors.Is(err, surveycore.ErrParseFailed) || !strings.Contains(err.Error(), survey.parseError) { + t.Fatalf("expected the fixture's unsupported-type error, got %v", err) + } + t.Log("live questions fetched; unsupported rating types correctly reported") + return + } + config := defaultConfigWithRetry(t, ctx, surveycore.New(), survey.url) + if config.SurveySource.Provider != survey.provider || config.SurveyDefinition.Provider != survey.provider { + t.Fatalf("provider mismatch: source=%q definition=%q", config.SurveySource.Provider, config.SurveyDefinition.Provider) + } + if strings.TrimSpace(config.Title) == "" || len(config.SurveyDefinition.Questions) == 0 || len(config.AnswerPlan.Strategies) == 0 { + t.Fatal("live parser returned an empty title, questions, or answer strategies") + } + }) + } +} + func TestLiveProviderSubmission(t *testing.T) { if os.Getenv("SURVEYCORE_LIVE") != "1" { t.Skip("set SURVEYCORE_LIVE=1 to enable authorized live submissions") diff --git a/README.md b/README.md index 41908b8..8090eb4 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,10 @@ api_key = "" | `POST` | `/api/v1/proxy/bonus` | 领取代理额度。 | | `POST` | `/api/v1/proxy/redeem` | 兑换代理卡。 | +## Python 配置导入 + +`POST /api/v1/tasks` 也接受 `surveycore-py` 的 `serialize_config()` 导出对象（`schema_version: 1`）。Go 本地配置可用 `config.Load()` 读取同一格式。支持范围和字段映射见 [数据结构](docs/sdk/schemas.md#python-配置导入)。 + ## 错误响应 API 错误统一返回稳定错误码、用户消息和调试详情： diff --git a/api/openapi.yaml b/api/openapi.yaml index d8af243..a0eb0b3 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -39,7 +39,7 @@ paths: '200': { description: Tasks ordered by creation time, content: { application/json: { schema: { $ref: '#/components/schemas/TaskList' } } } } post: operationId: createTask - requestBody: { required: true, content: { application/json: { schema: { $ref: '#/components/schemas/RunRequest' } } } } + requestBody: { required: true, content: { application/json: { schema: { oneOf: [{ $ref: '#/components/schemas/RunRequest' }, { $ref: '#/components/schemas/PythonConfig' }] } } } } responses: '202': { description: Task accepted, content: { application/json: { schema: { $ref: '#/components/schemas/Task' } } } } '400': { $ref: '#/components/responses/InvalidJSON' } @@ -167,6 +167,71 @@ components: answers: { $ref: '#/components/schemas/AnswerPlan' } reverseFill: { $ref: '#/components/schemas/ReverseFillPlan' } psychometrics: { $ref: '#/components/schemas/PsychometricPolicy' } + userAgent: { $ref: '#/components/schemas/UserAgentSettings' } + UserAgentSettings: + type: object + additionalProperties: false + properties: + enabled: { type: boolean } + ratios: { type: object, additionalProperties: { type: integer } } + PythonConfig: + description: Imports surveycore-py schema_version 1. Task responses use RunRequest; scheduling and results retain Go semantics. + type: object + additionalProperties: false + required: [schema_version, url] + properties: + schema_version: { type: integer, const: 1 } + url: { type: string, minLength: 1 } + provider: { type: [string, 'null'], enum: [wjx, qq, credamo, null] } + target: { type: integer, minimum: 1, default: 1 } + concurrency: { type: integer, minimum: 1, default: 1 } + questions: { type: array, items: { $ref: '#/components/schemas/PythonQuestionConfig' } } + answer_duration: { $ref: '#/components/schemas/IntegerPair', default: [60, 120] } + submit_interval: { $ref: '#/components/schemas/IntegerPair', default: [0, 0] } + answer_datetime_window: { $ref: '#/components/schemas/StringPair' } + random_user_agent: { type: boolean, default: false } + user_agent_ratios: { type: object, additionalProperties: { type: integer } } + reliability_mode: { type: boolean, default: true } + target_alpha: { type: number, default: 0.85 } + answer_rules: { type: array, items: { $ref: '#/components/schemas/ConsistencyRule' } } + PythonWeights: + anyOf: + - { type: 'null' } + - { type: number } + - type: array + items: + anyOf: + - { type: [number, 'null'] } + - { type: array, items: { type: [number, 'null'] } } + PythonQuestionConfig: + type: object + additionalProperties: false + required: [question_type] + properties: + question_type: { type: string, minLength: 1 } + probabilities: { $ref: '#/components/schemas/PythonWeights' } + custom_weights: { $ref: '#/components/schemas/PythonWeights' } + texts: { type: [array, 'null'], items: { type: string } } + rows: { type: integer } + option_count: { type: integer } + distribution_mode: { type: string } + question_num: { type: [integer, 'null'] } + question_title: { type: [string, 'null'] } + survey_provider: { type: string } + provider_question_id: { type: [string, 'null'] } + provider_page_id: { type: [string, 'null'] } + option_fill_texts: { type: [array, 'null'], items: { type: [string, 'null'] } } + fillable_option_indices: { type: [array, 'null'], items: { type: integer } } + attached_option_selects: { type: array, items: { $ref: '#/components/schemas/AttachedOptionSelect' } } + is_location: { type: boolean } + location_parts: { type: array, items: { type: string } } + multi_text_blank_modes: { type: array, items: { type: string } } + multi_text_blank_ai_flags: { type: array, items: { type: boolean } } + multi_text_blank_int_ranges: { type: array, items: { type: array, items: { type: integer } } } + text_random_mode: { type: string } + text_random_int_range: { type: array, items: { type: integer } } + dimension: { type: [string, 'null'] } + psycho_bias: { type: string } SurveySource: type: object additionalProperties: false diff --git a/docs/sdk/api.md b/docs/sdk/api.md index 6e1bb5a..cab556f 100644 --- a/docs/sdk/api.md +++ b/docs/sdk/api.md @@ -120,7 +120,9 @@ POST /api/v1/tasks Content-Type: application/json ``` -请求体必须使用 `RunRequest` 的嵌套结构。建议先获取默认配置，再修改目标数和并发数： +请求体接受 `RunRequest` 的嵌套结构，或 `surveycore-py` 导出的 `schema_version: 1` 配置。Python 配置会转换为 `RunRequest` 后创建任务，任务响应中的 `config` 使用 Go 的嵌套结构。字段映射见 [Python 配置导入](./schemas#python-配置导入)。 + +使用嵌套结构时，建议先获取默认配置，再修改目标数和并发数： ```json { diff --git a/docs/sdk/errors.md b/docs/sdk/errors.md index b786be9..05dfc00 100644 --- a/docs/sdk/errors.md +++ b/docs/sdk/errors.md @@ -45,7 +45,7 @@ outline: deep ## 注意事项 -- JSON 接口会拒绝未知字段。V1 任务请求必须使用 `source`、`definition`、`execution`、`answers`、`reverseFill` 和 `psychometrics` 结构。 +- JSON 接口会拒绝未知字段。V1 任务请求接受嵌套的 `RunRequest` 或 `schema_version: 1` 的 Python 配置；两种格式不能混用。Python 配置版本、字段类型、范围或权重无效时返回 `400 invalid_json`。 - 二维码上传字段名是 `file`。 - `202` 不代表提交成功。继续查询任务状态和日志。 - 任务运行期配置错误会写入任务的 `error` 字段，并把状态改成 `failed`。 diff --git a/docs/sdk/examples.md b/docs/sdk/examples.md index a7deaf3..31e25c0 100644 --- a/docs/sdk/examples.md +++ b/docs/sdk/examples.md @@ -115,6 +115,27 @@ task = request_json("POST", "/tasks", json=config) print(task["id"]) ``` +### 使用 surveycore-py 配置创建任务 + +```python +import httpx +from surveycore import QuestionConfig, RunConfig, serialize_config + +config = RunConfig( + url="https://www.wjx.cn/vm/example.aspx", + target=3, + concurrency=2, + questions=[QuestionConfig(question_type="single", probabilities=[0, 1])], +) +response = httpx.post( + "http://127.0.0.1:19178/api/v1/tasks", + json=serialize_config(config), + timeout=30, +) +response.raise_for_status() +print(response.json()["id"]) +``` + ### 上传二维码 JavaScript： diff --git a/docs/sdk/schemas.md b/docs/sdk/schemas.md index 6147681..c2d2aca 100644 --- a/docs/sdk/schemas.md +++ b/docs/sdk/schemas.md @@ -62,6 +62,7 @@ outline: deep | `answers.questions` | `QuestionStrategy[]` | 每道题的答案策略。 | | `reverseFill` | `ReverseFillPlan` | Excel 反填配置。 | | `psychometrics` | `PsychometricPolicy` | 信效度配置。 | +| `userAgent` | `object` | 可选随机 User-Agent 配置，包含 `enabled` 和 `ratios`（`wechat`、`mobile`、`pc` 权重）。 | ## QuestionStrategy @@ -139,3 +140,22 @@ outline: deep ## 本地配置文件 `configs/example.json` 使用 `schemaVersion: 2` 的 `ConfigDocument`。它包含 `survey`、`execution`、`network`、`answers`、`reverseFill` 和 `psychometrics` 六个区段。该文件格式与 REST `RunRequest` 不是同一个 JSON 外壳。 + +## Python 配置导入 + +`POST /api/v1/tasks`、`config.DeserializeRunRequest()` 和 `config.Load()` 接受 `surveycore-py` 的 `serialize_config()` 输出（`schema_version: 1`）。`config.DeserializePythonConfig()` 可先转换为本地 `ConfigDocument`。保存时仍写入 Go 的 v2 格式。 + +| Python 字段 | Go 字段 | +|---|---| +| `url` / `provider` | `source.url` / `source.provider`；省略或为 null 的 provider 由运行层识别。 | +| `target` / `concurrency` | `execution.target` / `execution.threads`。 | +| `answer_duration` / `submit_interval` / `answer_datetime_window` | 对应 `execution` 中的三个时间范围；保留原始范围，包括 `[0, 0]`。 | +| `questions` / `answer_rules` | `answers.questions` / `answers.rules`；省略或为 null 的题号按配置顺序从 1 编号。 | +| `random_user_agent` / `user_agent_ratios` | `userAgent.enabled` / `userAgent.ratios`，本地 v2 文件保存在 `network`。 | +| `reliability_mode` / `target_alpha` | `psychometrics.enabled` / `psychometrics.targetAlpha`。 | + +题目中的 `probabilities` 和 `custom_weights` 接受 Python 的数字、数组、矩阵数组或 null，转换为 Go 的 `WeightTable`。选择题的 null 或 `-1` 表示随机；多选权重保留 0–100 数值。支持其余 Go 已有的题目策略字段，并拒绝未知字段。 + +默认目标数和并发数都是 1，作答时长为 `[60, 120]`，间隔为 `[0, 0]`，信效度开关为 true、目标 alpha 为 0.85，随机 User-Agent 默认关闭。Python 序列化格式不包含 `answer_generator` 或题目 `ai_enabled`，导入会拒绝这些字段。 + +这一兼容入口用于配置导入。任务仍使用 Go 的重试、间隔和结果返回规则；Python 的逐次提交结果接口另行对齐。 diff --git a/internal/service/manager.go b/internal/service/manager.go index 731d4b1..af8875d 100644 --- a/internal/service/manager.go +++ b/internal/service/manager.go @@ -69,9 +69,10 @@ func (m *Manager) Create(ctx context.Context, cfg *model.RunRequest) (*Task, err m.tasks[id] = task m.runtimes[id] = &taskRuntime{cancel: cancel} m.wg.Add(1) + snapshot := cloneTask(task) m.mu.Unlock() go func() { defer m.wg.Done(); m.run(runCtx, id) }() - return cloneTask(task), nil + return snapshot, nil } func (m *Manager) run(ctx context.Context, id string) { diff --git a/pkg/restapi/server.go b/pkg/restapi/server.go index 231d175..05e2851 100644 --- a/pkg/restapi/server.go +++ b/pkg/restapi/server.go @@ -1,6 +1,7 @@ package restapi import ( + "bytes" "encoding/json" "errors" "fmt" @@ -13,6 +14,7 @@ import ( surveyio "github.com/SurveyController/SurveyCore/internal/io" "github.com/SurveyController/SurveyCore/internal/service" "github.com/SurveyController/SurveyCore/pkg/surveycore" + surveyconfig "github.com/SurveyController/SurveyCore/pkg/surveycore/config" "github.com/SurveyController/SurveyCore/pkg/surveycore/model" "github.com/SurveyController/SurveyCore/pkg/surveycore/proxy" ) @@ -126,8 +128,13 @@ func (s *Server) config(w http.ResponseWriter, r *http.Request) { write(w, 200, v) } func (s *Server) create(w http.ResponseWriter, r *http.Request) { - var cfg model.RunRequest - if err := decode(r, &cfg); err != nil { + var body json.RawMessage + if err := decode(r, &body); err != nil { + fail(w, 400, "invalid_json", err) + return + } + cfg, err := decodeTaskConfig(body) + if err != nil { fail(w, 400, "invalid_json", err) return } @@ -138,6 +145,26 @@ func (s *Server) create(w http.ResponseWriter, r *http.Request) { } write(w, 202, t) } + +func decodeTaskConfig(body []byte) (model.RunRequest, error) { + var payload map[string]any + payloadDecoder := json.NewDecoder(bytes.NewReader(body)) + payloadDecoder.UseNumber() + if err := payloadDecoder.Decode(&payload); err != nil { + return model.RunRequest{}, err + } + if payload == nil { + return model.RunRequest{}, errors.New("task config must be a JSON object") + } + if _, ok := payload["schema_version"]; ok { + return surveyconfig.DeserializeRunRequest(payload) + } + var config model.RunRequest + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + err := decoder.Decode(&config) + return config, err +} func (s *Server) list(w http.ResponseWriter, _ *http.Request) { write(w, 200, map[string]any{"tasks": s.manager.List()}) } diff --git a/pkg/restapi/server_test.go b/pkg/restapi/server_test.go index 5d71a1f..1bc32a9 100644 --- a/pkg/restapi/server_test.go +++ b/pkg/restapi/server_test.go @@ -1,12 +1,15 @@ package restapi import ( + "context" "encoding/json" "io" "net/http" "net/http/httptest" "strings" "testing" + + "github.com/SurveyController/SurveyCore/internal/service" ) func TestV1VersionUsesConfiguredBuildVersion(t *testing.T) { @@ -36,6 +39,53 @@ func TestV1VersionUsesConfiguredBuildVersion(t *testing.T) { } } +func TestV1CreateTaskAcceptsPythonConfig(t *testing.T) { + s, err := New(Config{DBPath: t.TempDir() + "/tasks.db"}) + if err != nil { + t.Fatal(err) + } + defer s.Close() + // A cancelled context exercises task creation and persistence without upstream requests. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + body := `{"schema_version":1,"url":"https://www.wjx.cn/vm/demo.aspx","target":3,"concurrency":2,"random_user_agent":true,"user_agent_ratios":{"pc":100},"questions":[{"question_type":"single","probabilities":[0,1]}]}` + req := httptest.NewRequest(http.MethodPost, "/api/v1/tasks", strings.NewReader(body)).WithContext(ctx) + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, req) + if rec.Code != http.StatusAccepted { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } + var task service.Task + if err := json.Unmarshal(rec.Body.Bytes(), &task); err != nil { + t.Fatal(err) + } + if task.Config.ExecutionPlan.Threads != 2 || task.Config.Target != 3 || task.Config.UserAgent == nil || !task.Config.UserAgent.Enabled || task.Config.AnswerPlan.Strategies[0].Probabilities.Options[1] != 1 { + t.Fatalf("task config = %#v", task.Config) + } +} + +func TestV1CreateTaskRejectsInvalidPythonConfig(t *testing.T) { + s, err := New(Config{DBPath: t.TempDir() + "/tasks.db"}) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for _, body := range []string{ + `{"schema_version":2,"url":"x"}`, + `{"schema_version":1,"url":"x","concurrency":0}`, + `{"schema_version":1,"url":"x","execution":{}}`, + `{"schema_version":1,"url":"x","questions":[{"question_type":"single","unknown":1}]}`, + `{"schema_version":1,"url":"x"} {}`, + `null`, + } { + rec := httptest.NewRecorder() + s.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/api/v1/tasks", strings.NewReader(body))) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), `"code":"invalid_json"`) { + t.Fatalf("body=%s status=%d error=%s", body, rec.Code, rec.Body.String()) + } + } +} + func TestV1HealthAndNotFound(t *testing.T) { dir := t.TempDir() s, err := New(Config{DBPath: dir + "\\test.db"}) diff --git a/pkg/surveycore/config/codec.go b/pkg/surveycore/config/codec.go index 56e9367..321cd23 100644 --- a/pkg/surveycore/config/codec.go +++ b/pkg/surveycore/config/codec.go @@ -48,8 +48,15 @@ func SerializeRunRequest(config model.RunRequest) map[string]any { return payload } -// DeserializeRunRequest accepts schemaVersion 2 and supported legacy objects. +// DeserializeRunRequest accepts local v2, Python v1, and supported legacy objects. func DeserializeRunRequest(payload map[string]any) (model.RunRequest, error) { + if _, ok := payload["schema_version"]; ok { + document, err := DeserializePythonConfig(payload) + if err != nil { + return model.RunRequest{}, err + } + return RunRequestFromConfigDocument(document) + } if _, ok := payload["schemaVersion"]; ok { document, err := DeserializeConfigDocument(payload) if err != nil { diff --git a/pkg/surveycore/config/doc.go b/pkg/surveycore/config/doc.go index 20cd4d2..c2f4d47 100644 --- a/pkg/surveycore/config/doc.go +++ b/pkg/surveycore/config/doc.go @@ -1,3 +1,4 @@ // Package config reads, validates, migrates, and atomically stores SurveyCore -// schemaVersion 2 local configuration documents. +// schemaVersion 2 local configuration documents, including imports from +// surveycore-py schema_version 1. package config diff --git a/pkg/surveycore/config/document.go b/pkg/surveycore/config/document.go index 5dd8aa2..b048616 100644 --- a/pkg/surveycore/config/document.go +++ b/pkg/surveycore/config/document.go @@ -49,7 +49,7 @@ type ConfigDocument struct { // ConfigDocumentFromRunRequest converts a public run request to a local document. func ConfigDocumentFromRunRequest(config model.RunRequest) ConfigDocument { - return ConfigDocument{ + document := ConfigDocument{ SchemaVersion: ConfigSchemaVersion, Survey: SurveyDocument{ URL: strings.TrimSpace(config.SurveySource.URL), @@ -62,6 +62,11 @@ func ConfigDocumentFromRunRequest(config model.RunRequest) ConfigDocument { ReverseFill: config.ReverseFillPlan, Psychometrics: config.PsychometricPolicy, } + if config.UserAgent != nil { + document.Network.RandomUAEnabled = config.UserAgent.Enabled + document.Network.RandomUARatios = cloneIntValues(config.UserAgent.Ratios) + } + return document } // RunRequestFromConfigDocument validates and converts a local document. @@ -75,14 +80,21 @@ func RunRequestFromConfigDocument(document ConfigDocument) (model.RunRequest, er definition := model.CloneSurveyDefinition(document.Survey.Definition) definition.Provider = firstNonEmpty(document.Survey.Provider, definition.Provider) definition.Title = firstNonEmpty(document.Survey.Title, definition.Title) - return model.RunRequest{ + request := model.RunRequest{ SurveySource: model.SurveySource{URL: strings.TrimSpace(document.Survey.URL), Provider: firstNonEmpty(document.Survey.Provider, definition.Provider)}, SurveyDefinition: definition, ExecutionPlan: document.Execution, AnswerPlan: model.CloneAnswerPlan(document.Answers), ReverseFillPlan: document.ReverseFill, PsychometricPolicy: document.Psychometrics, - }, nil + } + if document.Network.RandomUAEnabled || document.Network.RandomUARatios != nil { + request.UserAgent = &model.UserAgentSettings{ + Enabled: document.Network.RandomUAEnabled, + Ratios: cloneIntValues(document.Network.RandomUARatios), + } + } + return request, nil } func validateAnswerWeights(plan model.AnswerPlan) error { diff --git a/pkg/surveycore/config/python.go b/pkg/surveycore/config/python.go new file mode 100644 index 0000000..8e441d6 --- /dev/null +++ b/pkg/surveycore/config/python.go @@ -0,0 +1,200 @@ +package config + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "github.com/SurveyController/SurveyCore/pkg/surveycore/model" +) + +// pythonConfig mirrors the serializable RunConfig contract in surveycore-py. +// Callable answer generators and ai_enabled are intentionally not serialized there. +type pythonConfig struct { + SchemaVersion int `json:"schema_version"` + URL string `json:"url"` + Questions []json.RawMessage `json:"questions"` + Target int `json:"target"` + Concurrency int `json:"concurrency"` + Provider *string `json:"provider"` + AnswerDuration []int `json:"answer_duration"` + SubmitInterval []int `json:"submit_interval"` + AnswerDatetimeWindow []string `json:"answer_datetime_window"` + RandomUserAgent bool `json:"random_user_agent"` + UserAgentRatios map[string]int `json:"user_agent_ratios"` + ReliabilityMode bool `json:"reliability_mode"` + TargetAlpha float64 `json:"target_alpha"` + AnswerRules []model.ConsistencyRule `json:"answer_rules"` +} + +// DeserializePythonConfig imports a surveycore-py schema_version 1 object into +// the local document format, retaining user-agent settings for subsequent runs. +func DeserializePythonConfig(payload map[string]any) (ConfigDocument, error) { + data, err := json.Marshal(payload) + if err != nil { + return ConfigDocument{}, err + } + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil || raw == nil { + return ConfigDocument{}, fmt.Errorf("Python 配置必须是对象") + } + for field, value := range raw { + if field != "provider" && bytes.Equal(value, []byte("null")) { + return ConfigDocument{}, fmt.Errorf("Python 配置 %s 不能为 null", field) + } + } + config := pythonConfig{ + Target: 1, Concurrency: 1, + AnswerDuration: []int{60, 120}, SubmitInterval: []int{0, 0}, + AnswerDatetimeWindow: []string{"", ""}, + UserAgentRatios: map[string]int{"wechat": 33, "mobile": 33, "pc": 34}, + ReliabilityMode: true, TargetAlpha: 0.85, + } + if _, provided := raw["user_agent_ratios"]; provided { + config.UserAgentRatios = nil + } + if err := decodePythonObject(data, &config); err != nil { + return ConfigDocument{}, fmt.Errorf("Python 配置字段无效：%w", err) + } + if config.SchemaVersion != 1 { + return ConfigDocument{}, fmt.Errorf("不支持的 Python 配置版本：%d", config.SchemaVersion) + } + config.URL = strings.TrimSpace(config.URL) + if config.URL == "" || config.Target < 1 || config.Concurrency < 1 { + return ConfigDocument{}, fmt.Errorf("Python 配置 url 不能为空，target 和 concurrency 必须大于 0") + } + provider := "" + if config.Provider != nil { + provider = *config.Provider + if provider != model.ProviderWJX && provider != model.ProviderQQ && provider != model.ProviderCredamo { + return ConfigDocument{}, fmt.Errorf("Python 配置 provider 必须是 wjx、qq 或 credamo") + } + } + for field, values := range map[string][]int{"answer_duration": config.AnswerDuration, "submit_interval": config.SubmitInterval} { + if len(values) != 2 || values[0] < 0 || values[1] < values[0] || containsPythonNull(raw[field]) { + return ConfigDocument{}, fmt.Errorf("Python 配置 %s 必须是递增的两个非负整数", field) + } + } + if len(config.AnswerDatetimeWindow) != 2 || containsPythonNull(raw["answer_datetime_window"]) { + return ConfigDocument{}, fmt.Errorf("Python 配置 answer_datetime_window 必须包含两个字符串") + } + if containsPythonNull(raw["user_agent_ratios"]) || containsPythonNull(raw["answer_rules"]) { + return ConfigDocument{}, fmt.Errorf("Python 配置 user_agent_ratios 和 answer_rules 不能包含 null") + } + strategies := make([]model.QuestionStrategy, 0, len(config.Questions)) + for index, question := range config.Questions { + strategy, err := decodePythonQuestion(question, index+1) + if err != nil { + return ConfigDocument{}, fmt.Errorf("Python 配置 questions[%d]：%w", index+1, err) + } + strategies = append(strategies, strategy) + } + document := ConfigDocument{ + SchemaVersion: ConfigSchemaVersion, + Survey: SurveyDocument{URL: config.URL, Provider: provider, Definition: model.SurveyDefinition{Provider: provider}}, + Execution: model.ExecutionPlan{ + Target: config.Target, Threads: config.Concurrency, + AnswerDuration: [2]int{config.AnswerDuration[0], config.AnswerDuration[1]}, + SubmitInterval: [2]int{config.SubmitInterval[0], config.SubmitInterval[1]}, + AnswerDatetimeWindow: [2]string{config.AnswerDatetimeWindow[0], config.AnswerDatetimeWindow[1]}, + }, + Network: NetworkSettings{ProxySource: "default", RandomUAEnabled: config.RandomUserAgent, RandomUARatios: config.UserAgentRatios}, + Answers: model.AnswerPlan{Rules: config.AnswerRules, Strategies: strategies}, + ReverseFill: model.ReverseFillPlan{Format: ReverseFillFormatAuto, StartRow: 1, Threads: 1}, + Psychometrics: model.PsychometricPolicy{Enabled: config.ReliabilityMode, TargetAlpha: config.TargetAlpha}, + } + if _, err := RunRequestFromConfigDocument(document); err != nil { + return ConfigDocument{}, err + } + return document, nil +} + +func decodePythonQuestion(data []byte, index int) (model.QuestionStrategy, error) { + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil || raw == nil { + return model.QuestionStrategy{}, fmt.Errorf("题目必须是对象") + } + if _, ok := raw["ai_enabled"]; ok { + return model.QuestionStrategy{}, fmt.Errorf("ai_enabled 不属于 Python 序列化配置") + } + strategy := model.QuestionStrategy{ + Rows: 1, DistributionMode: "random", TextRandomMode: "none", + PsychoBias: "custom", SurveyProvider: model.ProviderWJX, + } + weights := raw["probabilities"] + custom := raw["custom_weights"] + delete(raw, "probabilities") + delete(raw, "custom_weights") + remaining, err := json.Marshal(raw) + if err != nil { + return strategy, err + } + if err := decodePythonObject(remaining, &strategy); err != nil { + return strategy, err + } + if strings.TrimSpace(string(strategy.QuestionType)) == "" { + return strategy, fmt.Errorf("question_type 必须是非空字符串") + } + if strategy.QuestionNum == nil { + strategy.QuestionNum = &index + } + if strategy.Probabilities, err = decodePythonWeights(weights, strategy.QuestionType); err != nil { + return strategy, fmt.Errorf("probabilities：%w", err) + } + if strategy.CustomWeights, err = decodePythonWeights(custom, strategy.QuestionType); err != nil { + return strategy, fmt.Errorf("custom_weights：%w", err) + } + return strategy, nil +} + +func decodePythonWeights(data []byte, kind model.QuestionKind) (model.WeightTable, error) { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + return model.WeightTable{}, nil + } + var value float64 + if json.Unmarshal(data, &value) == nil { + if value <= 0 && kind != model.QuestionKindSlider { + return model.WeightTable{}, nil + } + return model.OptionWeights(value), nil + } + var values []float64 + if json.Unmarshal(data, &values) == nil { + return model.OptionWeights(values...), nil + } + var rows [][]float64 + if err := json.Unmarshal(data, &rows); err != nil { + return model.WeightTable{}, fmt.Errorf("必须是数字、数组或矩阵权重") + } + return model.RowWeights(rows...), nil +} + +func decodePythonObject(data []byte, target any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + return decoder.Decode(target) +} + +func containsPythonNull(data []byte) bool { + var value any + if len(data) == 0 || json.Unmarshal(data, &value) != nil { + return false + } + switch values := value.(type) { + case []any: + for _, item := range values { + if item == nil { + return true + } + } + case map[string]any: + for _, item := range values { + if item == nil { + return true + } + } + } + return false +} diff --git a/pkg/surveycore/config/python_test.go b/pkg/surveycore/config/python_test.go new file mode 100644 index 0000000..9de71ad --- /dev/null +++ b/pkg/surveycore/config/python_test.go @@ -0,0 +1,162 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/SurveyController/SurveyCore/pkg/surveycore/model" +) + +func pythonFixture(t *testing.T) map[string]any { + t.Helper() + data, err := os.ReadFile("testdata/python-v1.json") + if err != nil { + t.Fatal(err) + } + var payload map[string]any + if err := json.Unmarshal(data, &payload); err != nil { + t.Fatal(err) + } + return payload +} + +func TestImportPythonSerializerFixture(t *testing.T) { + // Generated with surveycore-py serialize_config at commit 411ef6d. + request, err := DeserializeRunRequest(pythonFixture(t)) + if err != nil { + t.Fatal(err) + } + if request.SurveySource.URL != "https://www.wjx.cn/vm/demo.aspx" || request.SurveySource.Provider != model.ProviderWJX { + t.Fatalf("source = %#v", request.SurveySource) + } + if request.Target != 3 || request.ExecutionPlan.Threads != 2 || request.AnswerDuration != [2]int{} || request.SubmitInterval != [2]int{} || request.FailStop || request.PauseOnAliyunCaptcha { + t.Fatalf("execution = %#v", request.ExecutionPlan) + } + if !request.PsychometricPolicy.Enabled || request.TargetAlpha != 0.85 { + t.Fatalf("psychometrics = %#v", request.PsychometricPolicy) + } + if request.UserAgent == nil || !request.UserAgent.Enabled || request.UserAgent.Ratios["pc"] != 100 { + t.Fatalf("userAgent = %#v", request.UserAgent) + } + strategies := request.AnswerPlan.Strategies + if len(strategies) != 5 { + t.Fatalf("strategies = %#v", strategies) + } + for index, strategy := range strategies { + if strategy.QuestionNum == nil || *strategy.QuestionNum != index+1 { + t.Fatalf("strategy %d is not bound to its implicit question number", index) + } + } + if !reflect.DeepEqual(strategies[0].Probabilities.Options, []float64{0, 1}) || !reflect.DeepEqual(strategies[1].Probabilities.Options, []float64{100, 0, 100}) || !reflect.DeepEqual(strategies[2].Probabilities.Rows, [][]float64{{1, 0}, {0, 1}}) || !reflect.DeepEqual(strategies[3].Texts, []string{"first", "second"}) || !reflect.DeepEqual(strategies[4].Probabilities.Options, []float64{37}) { + t.Fatalf("imported strategies = %#v", strategies) + } +} + +func TestPythonConfigDefaultsAndRandomWeights(t *testing.T) { + request, err := DeserializeRunRequest(map[string]any{ + "schema_version": 1, "url": " https://wj.qq.com/s2/123/hash/ ", "provider": nil, + "questions": []any{map[string]any{"question_type": "single", "probabilities": -1}}, + }) + if err != nil { + t.Fatal(err) + } + if request.Target != 1 || request.ExecutionPlan.Threads != 1 || request.AnswerDuration != [2]int{60, 120} || request.UserAgent.Enabled || request.UserAgent.Ratios["pc"] != 34 { + t.Fatalf("defaults = %#v", request) + } + if request.SurveySource.Provider != "" || len(request.AnswerPlan.Strategies[0].Probabilities.Options) != 0 { + t.Fatalf("auto provider/random weights = %#v", request) + } +} + +func TestPythonUserAgentRatiosReplaceDefaults(t *testing.T) { + request, err := DeserializeRunRequest(map[string]any{ + "schema_version": 1, "url": "x", + "user_agent_ratios": map[string]int{"pc": 100}, + }) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(request.UserAgent.Ratios, map[string]int{"pc": 100}) { + t.Fatalf("custom ratios were merged with defaults: %#v", request.UserAgent.Ratios) + } +} + +func TestPythonConfigRejectsInvalidInputs(t *testing.T) { + cases := map[string]any{ + "schema_version": 2, "unknown": true, "target": 0, "concurrency": "2", + "provider": "tencent", "url": " ", "questions": nil, + "answer_duration": []any{1, nil}, "submit_interval": []int{3, 2}, + "answer_datetime_window": []string{""}, "random_user_agent": 1, + "user_agent_ratios": map[string]any{"pc": nil}, "reliability_mode": nil, + "target_alpha": true, "answer_rules": []any{nil}, + "answer_generator": "callable", "schemaVersion": 2, + } + for field, value := range cases { + t.Run(field, func(t *testing.T) { + payload := map[string]any{"schema_version": 1, "url": "https://www.wjx.cn/vm/demo.aspx"} + payload[field] = value + if _, err := DeserializeRunRequest(payload); err == nil { + t.Fatalf("accepted %s=%#v", field, value) + } + }) + } + for _, question := range []any{ + nil, map[string]any{}, map[string]any{"question_type": ""}, + map[string]any{"question_type": "single", "probabilities": map[string]any{"options": []int{1}}}, + map[string]any{"question_type": "single", "probabilities": []int{0, 0}}, + map[string]any{"question_type": "single", "ai_enabled": true}, + map[string]any{"question_type": "single", "unknown": true}, + } { + if _, err := DeserializeRunRequest(map[string]any{"schema_version": 1, "url": "x", "questions": []any{question}}); err == nil { + t.Fatalf("accepted invalid question %#v", question) + } + } +} + +func TestLoadPythonConfigAndSaveAsLocalV2(t *testing.T) { + data, err := os.ReadFile("testdata/python-v1.json") + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "config.json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + request, err := Load(path, true) + if err != nil { + t.Fatal(err) + } + if _, err := Save(request, path); err != nil { + t.Fatal(err) + } + restored, err := Load(path, true) + if err != nil { + t.Fatal(err) + } + expectedAnswers, err := json.Marshal(request.AnswerPlan) + if err != nil { + t.Fatal(err) + } + restoredAnswers, err := json.Marshal(restored.AnswerPlan) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(request.UserAgent, restored.UserAgent) || string(expectedAnswers) != string(restoredAnswers) || request.ExecutionPlan != restored.ExecutionPlan { + t.Fatalf("local v2 round trip lost Python settings: %#v", restored) + } + // Native task persistence must retain the imported settings too. + encoded, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + var persisted model.RunRequest + if err := json.Unmarshal(encoded, &persisted); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(request.UserAgent, persisted.UserAgent) { + t.Fatalf("task JSON lost user-agent settings: %#v", persisted.UserAgent) + } +} diff --git a/pkg/surveycore/config/store.go b/pkg/surveycore/config/store.go index 6f2c224..a9723bb 100644 --- a/pkg/surveycore/config/store.go +++ b/pkg/surveycore/config/store.go @@ -17,7 +17,7 @@ func BuildDefaultConfigFilename(surveyTitle string) string { return sanitizeFilename(surveyTitle) + ".json" } -// Load reads either a schemaVersion 2 document or a supported legacy document. +// Load reads local v2, Python v1, or supported legacy configuration. func Load(path string, strict bool) (model.RunRequest, error) { document, err := LoadDocument(path, strict) if err != nil { @@ -65,7 +65,12 @@ func LoadDocument(path string, strict bool) (ConfigDocument, error) { } return document, nil } - document, err := migrateLegacyDocument(payload) + var document ConfigDocument + if _, ok := payload["schema_version"]; ok { + document, err = DeserializePythonConfig(payload) + } else { + document, err = migrateLegacyDocument(payload) + } if err != nil { if strict { return ConfigDocument{}, fmt.Errorf("配置不兼容: %s -> %w", path, err) diff --git a/pkg/surveycore/config/testdata/python-v1.json b/pkg/surveycore/config/testdata/python-v1.json new file mode 100644 index 0000000..8b03b53 --- /dev/null +++ b/pkg/surveycore/config/testdata/python-v1.json @@ -0,0 +1,182 @@ +{ + "schema_version": 1, + "url": "https://www.wjx.cn/vm/demo.aspx", + "questions": [ + { + "attached_option_selects": [], + "custom_weights": null, + "dimension": null, + "distribution_mode": "random", + "fillable_option_indices": null, + "is_location": false, + "location_parts": [], + "multi_text_blank_ai_flags": [], + "multi_text_blank_int_ranges": [], + "multi_text_blank_modes": [], + "option_count": 2, + "option_fill_texts": null, + "probabilities": [ + 0, + 1 + ], + "provider_page_id": null, + "provider_question_id": null, + "psycho_bias": "custom", + "question_num": null, + "question_title": null, + "question_type": "single", + "rows": 1, + "survey_provider": "wjx", + "text_random_int_range": [], + "text_random_mode": "none", + "texts": null + }, + { + "attached_option_selects": [], + "custom_weights": null, + "dimension": null, + "distribution_mode": "random", + "fillable_option_indices": null, + "is_location": false, + "location_parts": [], + "multi_text_blank_ai_flags": [], + "multi_text_blank_int_ranges": [], + "multi_text_blank_modes": [], + "option_count": 3, + "option_fill_texts": null, + "probabilities": [ + 100, + 0, + 100 + ], + "provider_page_id": null, + "provider_question_id": null, + "psycho_bias": "custom", + "question_num": null, + "question_title": null, + "question_type": "multiple", + "rows": 1, + "survey_provider": "wjx", + "text_random_int_range": [], + "text_random_mode": "none", + "texts": null + }, + { + "attached_option_selects": [], + "custom_weights": null, + "dimension": null, + "distribution_mode": "random", + "fillable_option_indices": null, + "is_location": false, + "location_parts": [], + "multi_text_blank_ai_flags": [], + "multi_text_blank_int_ranges": [], + "multi_text_blank_modes": [], + "option_count": 2, + "option_fill_texts": null, + "probabilities": [ + [ + 1, + 0 + ], + [ + 0, + 1 + ] + ], + "provider_page_id": null, + "provider_question_id": null, + "psycho_bias": "custom", + "question_num": null, + "question_title": null, + "question_type": "matrix", + "rows": 2, + "survey_provider": "wjx", + "text_random_int_range": [], + "text_random_mode": "none", + "texts": null + }, + { + "attached_option_selects": [], + "custom_weights": null, + "dimension": null, + "distribution_mode": "random", + "fillable_option_indices": null, + "is_location": false, + "location_parts": [], + "multi_text_blank_ai_flags": [], + "multi_text_blank_int_ranges": [], + "multi_text_blank_modes": [], + "option_count": 0, + "option_fill_texts": null, + "probabilities": [ + 0, + 1 + ], + "provider_page_id": null, + "provider_question_id": null, + "psycho_bias": "custom", + "question_num": null, + "question_title": null, + "question_type": "text", + "rows": 1, + "survey_provider": "wjx", + "text_random_int_range": [], + "text_random_mode": "none", + "texts": [ + "first", + "second" + ] + }, + { + "attached_option_selects": [], + "custom_weights": null, + "dimension": null, + "distribution_mode": "custom", + "fillable_option_indices": null, + "is_location": false, + "location_parts": [], + "multi_text_blank_ai_flags": [], + "multi_text_blank_int_ranges": [], + "multi_text_blank_modes": [], + "option_count": 0, + "option_fill_texts": null, + "probabilities": 37, + "provider_page_id": null, + "provider_question_id": null, + "psycho_bias": "custom", + "question_num": null, + "question_title": null, + "question_type": "slider", + "rows": 1, + "survey_provider": "wjx", + "text_random_int_range": [], + "text_random_mode": "none", + "texts": null + } + ], + "target": 3, + "concurrency": 2, + "provider": "wjx", + "answer_duration": [ + 0, + 0 + ], + "submit_interval": [ + 0, + 0 + ], + "answer_datetime_window": [ + "", + "" + ], + "random_user_agent": true, + "user_agent_ratios": { + "wechat": 0, + "mobile": 0, + "pc": 100 + }, + "reliability_mode": true, + "target_alpha": 0.85, + "answer_rules": [] +} diff --git a/pkg/surveycore/internal/providers/tencent/client.go b/pkg/surveycore/internal/providers/tencent/client.go index f4c82b6..e252682 100644 --- a/pkg/surveycore/internal/providers/tencent/client.go +++ b/pkg/surveycore/internal/providers/tencent/client.go @@ -32,9 +32,13 @@ func (p Parser) Parse(ctx context.Context, surveyURL string) (model.SurveyDefini func (p Parser) fetchViaHTTP(ctx context.Context, surveyID string, hashValue string) (model.SurveyDefinition, error) { headers := apiHeaders(pageURL(surveyID, hashValue), p.UserAgent) - if _, err := p.requestAPI(ctx, surveyID, "session", hashValue, headers, nil); err != nil { + session, err := p.requestAPI(ctx, surveyID, "session", hashValue, headers, nil) + if err != nil { return model.SurveyDefinition{}, err } + if sessionID := strings.TrimSpace(stringValue(session["answer_session_id"])); sessionID != "" { + headers["X-Answer-Session"] = sessionID + } var lastErr error for _, locale := range locales { diff --git a/pkg/surveycore/internal/providers/tencent/parser_test.go b/pkg/surveycore/internal/providers/tencent/parser_test.go index 18377c9..cfe90e8 100644 --- a/pkg/surveycore/internal/providers/tencent/parser_test.go +++ b/pkg/surveycore/internal/providers/tencent/parser_test.go @@ -52,6 +52,35 @@ func TestParserParsesTencentQuestions(t *testing.T) { } } +func TestParserCarriesAnswerSessionToMetadataRequests(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/session") { + writeTencentJSON(t, w, map[string]any{"code": "OK", "data": map[string]any{"answer_session_id": "parse-session"}}) + return + } + if r.Header.Get("X-Answer-Session") != "parse-session" { + writeTencentJSON(t, w, map[string]any{"code": "InvalidArgument"}) + return + } + switch { + case strings.HasSuffix(r.URL.Path, "/meta"): + writeTencentJSON(t, w, map[string]any{"code": "OK", "data": map[string]any{"title": "Session survey"}}) + case strings.HasSuffix(r.URL.Path, "/questions"): + writeTencentJSON(t, w, map[string]any{"code": "OK", "data": map[string]any{"questions": loadTencentQuestionsFixture(t)}}) + default: + t.Errorf("unexpected path: %s", r.URL.Path) + } + })) + defer server.Close() + definition, err := (Parser{HTTP: rewriteClient(server.URL)}).Parse(context.Background(), "https://wj.qq.com/s2/123/hash/") + if err != nil { + t.Fatal(err) + } + if len(definition.Questions) == 0 { + t.Fatal("session-backed parser returned no questions") + } +} + func TestTencentAttachLogicMetadata(t *testing.T) { raw := []map[string]any{ {"id": "q-1", "type": "radio", "title": "单选", "page_id": "p-1", "page": 1, "options": []any{map[string]any{"id": "o-1", "text": "A", "display": "q-2"}}}, diff --git a/pkg/surveycore/model/run_types.go b/pkg/surveycore/model/run_types.go index 7ac9190..185e35a 100644 --- a/pkg/surveycore/model/run_types.go +++ b/pkg/surveycore/model/run_types.go @@ -30,8 +30,8 @@ type PsychometricPolicy struct { // UserAgentSettings controls randomized runtime user-agent selection. type UserAgentSettings struct { - Enabled bool - Ratios map[string]int + Enabled bool `json:"enabled"` + Ratios map[string]int `json:"ratios,omitempty"` } // RunRequest is the complete configuration accepted by Run and the V1 task API. @@ -42,6 +42,7 @@ type RunRequest struct { AnswerPlan `json:"answers"` ReverseFillPlan `json:"reverseFill"` PsychometricPolicy `json:"psychometrics"` + UserAgent *UserAgentSettings `json:"userAgent,omitempty"` } // AnswerAction is a prepared provider-neutral answer for one question. diff --git a/pkg/surveycore/runtime/answer_plan_clone.go b/pkg/surveycore/runtime/answer_plan_clone.go index 0553303..7620dd5 100644 --- a/pkg/surveycore/runtime/answer_plan_clone.go +++ b/pkg/surveycore/runtime/answer_plan_clone.go @@ -12,6 +12,14 @@ func cloneRunRequest(cfg *RunRequest) RunRequest { cloned := *cfg cloned.SurveyDefinition = model.CloneSurveyDefinition(cfg.SurveyDefinition) cloned.AnswerPlan = model.CloneAnswerPlan(cfg.AnswerPlan) + if cfg.UserAgent != nil { + settings := *cfg.UserAgent + settings.Ratios = make(map[string]int, len(cfg.UserAgent.Ratios)) + for key, value := range cfg.UserAgent.Ratios { + settings.Ratios[key] = value + } + cloned.UserAgent = &settings + } return cloned } diff --git a/pkg/surveycore/runtime/defaults.go b/pkg/surveycore/runtime/defaults.go index e7ae7da..6fd5d37 100644 --- a/pkg/surveycore/runtime/defaults.go +++ b/pkg/surveycore/runtime/defaults.go @@ -80,7 +80,7 @@ func populateConfigSurveyDefinition(cfg *RunRequest, definition *SurveyDefinitio } cfg.SurveyDefinition = *definition cfg.SurveySource.Provider = definition.Provider - cfg.AnswerPlan.Strategies = buildDefaultQuestionStrategies(definition.Questions) + ensureQuestionStrategies(cfg) } func ensureQuestionStrategies(cfg *RunRequest) { diff --git a/pkg/surveycore/runtime/python_config_test.go b/pkg/surveycore/runtime/python_config_test.go new file mode 100644 index 0000000..5245ebd --- /dev/null +++ b/pkg/surveycore/runtime/python_config_test.go @@ -0,0 +1,81 @@ +package runtime + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + + surveyconfig "github.com/SurveyController/SurveyCore/pkg/surveycore/config" +) + +func TestRunImportedPythonConfigPreservesAnswersAndUserAgent(t *testing.T) { + data, err := os.ReadFile("../config/testdata/python-v1.json") + if err != nil { + t.Fatal(err) + } + var payload map[string]any + if err := json.Unmarshal(data, &payload); err != nil { + t.Fatal(err) + } + request, err := surveyconfig.DeserializeRunRequest(payload) + if err != nil { + t.Fatal(err) + } + var mu sync.Mutex + var submissions []string + var agents []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/vm/demo.aspx": + _, _ = w.Write([]byte(`Python fixture
+
Choice
A
B
+
`)) + case "/joinnew/processjq.ashx": + if err := r.ParseForm(); err != nil { + t.Error(err) + } + mu.Lock() + submissions = append(submissions, r.Form.Get("submitdata")) + agents = append(agents, r.UserAgent()) + mu.Unlock() + _, _ = w.Write([]byte("10")) + default: + t.Errorf("unexpected path: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + result, err := New(WithHTTPClient(rewriteWJXHTTPClient(server.URL))).Run(context.Background(), &request) + if err != nil { + t.Fatal(err) + } + if result.Success != 3 || result.Fail != 0 { + t.Fatalf("result = %#v", result) + } + mu.Lock() + defer mu.Unlock() + if len(submissions) != 3 { + t.Fatalf("submissions = %#v", submissions) + } + for _, answer := range submissions { + if answer != "1$2" { + t.Fatalf("imported [0,1] weights were overwritten: %q", answer) + } + } + for _, agent := range agents { + if !strings.Contains(agent, "Windows NT 10.0") || !strings.Contains(agent, "Chrome/121") { + t.Fatalf("unexpected user-agent profile: %q", agent) + } + } + if len(agents) != 3 { + t.Fatalf("requests = %#v", agents) + } + if options := ExecutionOptionsFromConfig(&request); !options.UserAgent.Enabled || options.UserAgent.Ratios["pc"] != 100 || options.Threads != 2 { + t.Fatalf("execution options = %#v", options) + } +} diff --git a/pkg/surveycore/runtime/run_options.go b/pkg/surveycore/runtime/run_options.go index 8318dff..72621c1 100644 --- a/pkg/surveycore/runtime/run_options.go +++ b/pkg/surveycore/runtime/run_options.go @@ -23,7 +23,7 @@ func ExecutionOptionsFromConfig(cfg *RunRequest) ExecutionOptions { if cfg.PsychometricPolicy.Enabled { maxRetries = 1 } - return ExecutionOptions{ + options := ExecutionOptions{ Target: target, Threads: threads, MaxRetries: maxRetries, @@ -31,6 +31,14 @@ func ExecutionOptionsFromConfig(cfg *RunRequest) ExecutionOptions { FailStopThreshold: defaultFailStopThreshold, CooldownOnError: 30 * time.Second, } + if cfg.UserAgent != nil { + options.UserAgent = *cfg.UserAgent + options.UserAgent.Ratios = make(map[string]int, len(cfg.UserAgent.Ratios)) + for key, value := range cfg.UserAgent.Ratios { + options.UserAgent.Ratios[key] = value + } + } + return options } func failStopThreshold(options ExecutionOptions) int { diff --git a/tests/consumer/sdk_test.go b/tests/consumer/sdk_test.go index 3f3f580..866bbe0 100644 --- a/tests/consumer/sdk_test.go +++ b/tests/consumer/sdk_test.go @@ -37,6 +37,15 @@ func TestPublicPackagesCompileForExternalConsumer(t *testing.T) { if document.SchemaVersion != config.ConfigSchemaVersion { t.Fatalf("schema version = %d", document.SchemaVersion) } + imported, err := config.DeserializeRunRequest(map[string]any{ + "schema_version": 1, "url": "https://www.wjx.cn/vm/demo.aspx", "concurrency": 2, + }) + if err != nil { + t.Fatal(err) + } + if imported.ExecutionPlan.Threads != 2 { + t.Fatalf("Python concurrency = %d", imported.ExecutionPlan.Threads) + } pool := proxy.NewPool(proxy.PoolOptions{}) if pool == nil {