diff --git a/docs/command-surface.md b/docs/command-surface.md index d376ded0..55992699 100644 --- a/docs/command-surface.md +++ b/docs/command-surface.md @@ -193,6 +193,15 @@ rc paywalls delete [--force] # attached/published pa rc experiments list [--status ] rc experiments show # human detail view shows variants, targeting, placements, and chronological timeline rc experiments results [--platform

] [--country ] [--exposure-status exposed|not_exposed] [--currency ] # human totals and decision signals; unknown platforms warn before passing through +# Enrollment is 1–100 in the development API and backend validator; the dashboard guide currently says 10% minimum. +rc experiments create (--name --control --treatment --enrollment | --config ) +rc experiments duplicate [--name ] # copy configuration into a new draft; never copies results or status +rc experiments update --config # partial update; file or - for stdin +rc experiments delete # draft only; confirmation/--yes +rc experiments start # approval gate: confirmation/--yes; shows current configuration first +rc experiments pause # pauses enrollment; configuration review + confirmation/--yes +rc experiments resume # resumes enrollment; configuration review + confirmation/--yes +rc experiments stop # permanent; confirmation/--yes # Rico (AI assistant) rc rico [message] # streaming chat window in a TTY (--plain for a line loop) diff --git a/docs/previews/experiments-show.svg b/docs/previews/experiments-show.svg index 16bba185..14f030d7 100644 --- a/docs/previews/experiments-show.svg +++ b/docs/previews/experiments-show.svg @@ -1,6 +1,6 @@ - + + $ rc experiments show exp_snap --no-input --project-id proj_snap --api-key sk_snap ▍ Price — paywall -  exp_snap · paused +  exp_snap · draft Variants -  Control:    ofrng_control (Control) -  Treatment:  ofrng_treatment (Treatment) - -Audience -  Condition 1:  platform in ios - -Placements -  onboarding:  Control ofrng_control · Treatment ofrng_treatment - -Setup -  Enrollment:         50% -  Primary metric:     initial_conversion_rate -  Secondary metrics:  realized_ltv_per_customer, trial_conversion_rate -  Duration settings:  chance to win 95% - -Timeline -  Running time:  23d 0h 43m -  Use --json for the complete API response. +  Control:      ofrng_control (Control) · paywall pw_control +  Treatment:    ofrng_treatment (Treatment) · paywall pw_treatment +  Treatment C:  ofrng_extra + +Audience +  Condition 1:  platform in ios + +Placements +  onboarding:  Control ofrng_control · Treatment ofrng_treatment + +Setup +  Enrollment:         50% +  Enrollment mode:    only_new +  Type:               paywall_design +  Primary metric:     initial_conversion_rate +  Secondary metrics:  realized_ltv_per_customer, trial_conversion_rate +  Duration settings:  chance to win 95% + +Timeline +  Running time:  23d 0h 43m + +Conflicts +  Experiments:  exp_other +  Use --json for the complete API response. diff --git a/docs/previews/experiments-start.svg b/docs/previews/experiments-start.svg new file mode 100644 index 00000000..fb784d22 --- /dev/null +++ b/docs/previews/experiments-start.svg @@ -0,0 +1,45 @@ + + + + + +$ rc experiments start exp_snap --yes --no-input --project-id proj_snap --api-key sk_snap + +▍ Experiment — Price — paywall +  Start enrolling eligible customers in the experiment. + +  Status                      draft +  Control                     ofrng_control (Control) · paywall pw_control +  Treatment                   ofrng_treatment (Treatment) · paywall pw_treatment +  Treatment C                 ofrng_extra +  Enrollment                  50% +  Condition 1                 platform in ios +  Placement onboarding        Control ofrng_control · Treatment ofrng_treatment +  Type                        paywall_design +  Primary metric              initial_conversion_rate +  Secondary metrics           realized_ltv_per_customer, trial_conversion_rate +  Enrollment mode             only_new +  Duration                    chance to win 95% + +▐ Experiment conflicts: exp_other + + +▍ Plan +· 1. Start the experiment and enroll eligible customers +· Starting experiment… +✓ Started experiment exp_snap +▍ New paywall + +  ID:      exp_snap +  Status:  running +  rc experiments results exp_snap + + + diff --git a/docs/previews/experiments-stop.svg b/docs/previews/experiments-stop.svg new file mode 100644 index 00000000..3a60454f --- /dev/null +++ b/docs/previews/experiments-stop.svg @@ -0,0 +1,57 @@ + + + + + +$ rc experiments stop exp_snap --yes --no-input --project-id proj_snap --api-key sk_snap + +▍ Experiment — Price — paywall +  Permanently stop the experiment. + +  Status                      draft +  Control                     ofrng_control (Control) · paywall pw_control +  Treatment                   ofrng_treatment (Treatment) · paywall pw_treatment +  Treatment C                 ofrng_extra +  Enrollment                  50% +  Condition 1                 platform in ios +  Placement onboarding        Control ofrng_control · Treatment ofrng_treatment +  Type                        paywall_design +  Primary metric              initial_conversion_rate +  Secondary metrics           realized_ltv_per_customer, trial_conversion_rate +  Enrollment mode             only_new +  Duration                    chance to win 95% + +▐ Experiment conflicts: exp_other + + +▍ Plan +· 1. Stop the experiment permanently +✓ Stopped experiment exp_snap +▍ Price paywall +  exp_snap · stopped + +Variants +  none + +Audience +  Audience:  All eligible customers + +Placements +  No placement overrides + +Setup +  none + +Timeline +  Created:   +  Use --json for the complete API response. + + + diff --git a/docs/specs/cli-coverage.yaml b/docs/specs/cli-coverage.yaml index ae82f6c1..9b2a28d8 100644 --- a/docs/specs/cli-coverage.yaml +++ b/docs/specs/cli-coverage.yaml @@ -225,8 +225,22 @@ endpoints: # Experiments (development-status; provided by the beta overlay) - method: GET path: /projects/{project_id}/experiments + - method: POST + path: /projects/{project_id}/experiments - method: GET path: /projects/{project_id}/experiments/{experiment_id} + - method: POST + path: /projects/{project_id}/experiments/{experiment_id} + - method: DELETE + path: /projects/{project_id}/experiments/{experiment_id} + - method: POST + path: /projects/{project_id}/experiments/{experiment_id}/actions/start + - method: POST + path: /projects/{project_id}/experiments/{experiment_id}/actions/pause + - method: POST + path: /projects/{project_id}/experiments/{experiment_id}/actions/resume + - method: POST + path: /projects/{project_id}/experiments/{experiment_id}/actions/stop - method: GET path: /projects/{project_id}/experiments/{experiment_id}/results diff --git a/docs/specs/v2-beta-overlay.yaml b/docs/specs/v2-beta-overlay.yaml index 5cb9ee3a..bca33fd1 100644 --- a/docs/specs/v2-beta-overlay.yaml +++ b/docs/specs/v2-beta-overlay.yaml @@ -936,6 +936,16 @@ paths: responses: '200': description: OK + post: + summary: Create an experiment + operationId: create-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK /projects/{project_id}/experiments/{experiment_id}: parameters: - name: project_id @@ -948,6 +958,16 @@ paths: required: true schema: type: string + delete: + summary: Delete an experiment + operationId: delete-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK get: summary: Get an experiment operationId: get-experiment @@ -958,6 +978,104 @@ paths: responses: '200': description: OK + post: + summary: Update an experiment + operationId: update-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}/actions/pause: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + post: + summary: Pause an experiment + operationId: pause-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}/actions/resume: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + post: + summary: Resume an experiment + operationId: resume-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}/actions/start: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + post: + summary: Start an experiment + operationId: start-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}/actions/stop: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + post: + summary: Stop an experiment + operationId: stop-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK /projects/{project_id}/experiments/{experiment_id}/results: parameters: - name: project_id diff --git a/internal/api/experiments.go b/internal/api/experiments.go index ee82b3c3..9481eeae 100644 --- a/internal/api/experiments.go +++ b/internal/api/experiments.go @@ -197,6 +197,64 @@ type ExperimentResultsOptions struct { Currency string } +type ExperimentCreate struct { + DisplayName string `json:"display_name"` + EnrollmentPercentage int `json:"enrollment_percentage"` + OfferingAID string `json:"offering_a_id"` + OfferingBID string `json:"offering_b_id"` + OfferingCID *string `json:"offering_c_id,omitempty"` + OfferingDID *string `json:"offering_d_id,omitempty"` + TargetingConditions json.RawMessage `json:"targeting_conditions,omitempty"` + AudienceID *string `json:"audience_id,omitempty"` + Placements json.RawMessage `json:"placements,omitempty"` + Notes *string `json:"notes,omitempty"` + ExperimentType *string `json:"experiment_type,omitempty"` + PrimaryMetric *string `json:"primary_metric,omitempty"` + SecondaryMetrics []string `json:"secondary_metrics,omitempty"` + EnrollmentMode *string `json:"enrollment_mode,omitempty"` + ExperimentDurationSettings json.RawMessage `json:"experiment_duration_settings,omitempty"` +} + +type ExperimentUpdate map[string]json.RawMessage + +func (s *ExperimentsService) Create(ctx context.Context, projectID string, body ExperimentCreate) (*Experiment, error) { + var out Experiment + err := s.c.do(ctx, http.MethodPost, pathExperiments(projectID), body, &out) + return &out, err +} + +func (s *ExperimentsService) Update(ctx context.Context, projectID, id string, body ExperimentUpdate) (*Experiment, error) { + var out Experiment + err := s.c.do(ctx, http.MethodPost, pathExperiment(projectID, id), body, &out) + return &out, err +} + +func (s *ExperimentsService) Delete(ctx context.Context, projectID, id string) error { + return s.c.do(ctx, http.MethodDelete, pathExperiment(projectID, id), nil, nil) +} + +func (s *ExperimentsService) Start(ctx context.Context, projectID, id string) (*Experiment, error) { + return s.action(ctx, pathExperimentActionsStart(projectID, id)) +} + +func (s *ExperimentsService) Pause(ctx context.Context, projectID, id string) (*Experiment, error) { + return s.action(ctx, pathExperimentActionsPause(projectID, id)) +} + +func (s *ExperimentsService) Resume(ctx context.Context, projectID, id string) (*Experiment, error) { + return s.action(ctx, pathExperimentActionsResume(projectID, id)) +} + +func (s *ExperimentsService) Stop(ctx context.Context, projectID, id string) (*Experiment, error) { + return s.action(ctx, pathExperimentActionsStop(projectID, id)) +} + +func (s *ExperimentsService) action(ctx context.Context, path string) (*Experiment, error) { + var out Experiment + err := s.c.do(ctx, http.MethodPost, path, nil, &out) + return &out, err +} + func (s *ExperimentsService) Results(ctx context.Context, projectID, id string, opts ExperimentResultsOptions) (*ExperimentResults, error) { q := url.Values{} if opts.Platform != "" { diff --git a/internal/api/experiments_test.go b/internal/api/experiments_test.go index 6df2b4a6..1f0db8c6 100644 --- a/internal/api/experiments_test.go +++ b/internal/api/experiments_test.go @@ -2,6 +2,7 @@ package api_test import ( "context" + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -10,6 +11,59 @@ import ( "github.com/revenuecat/cli/internal/api" ) +func TestExperimentLifecycleRoutes(t *testing.T) { + requests := []string{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Method+" "+r.URL.Path) + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodPost && r.URL.Path == "/projects/proj/experiments" { + var body api.ExperimentCreate + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if body.OfferingAID != "ofrng_a" || body.OfferingBID != "ofrng_b" || body.EnrollmentPercentage != 50 { + t.Fatalf("unexpected create request: %+v", body) + } + } + if r.Method == http.MethodDelete { + fmt.Fprint(w, `{"object":"deleted_object","id":"exp1"}`) + return + } + fmt.Fprint(w, `{"object":"experiment","id":"exp1","display_name":"Test","status":"draft","created_at":1,"updated_at":2}`) + })) + t.Cleanup(srv.Close) + client := api.NewClient(api.Options{APIKey: "sk_test", BaseURL: srv.URL}) + ctx := context.Background() + _, err := client.Experiments.Create(ctx, "proj", api.ExperimentCreate{DisplayName: "Test", OfferingAID: "ofrng_a", OfferingBID: "ofrng_b", EnrollmentPercentage: 50}) + if err != nil { + t.Fatal(err) + } + _, err = client.Experiments.Update(ctx, "proj", "exp1", api.ExperimentUpdate{"notes": json.RawMessage(`"reviewed"`)}) + if err != nil { + t.Fatal(err) + } + for _, action := range []func(context.Context, string, string) (*api.Experiment, error){client.Experiments.Start, client.Experiments.Pause, client.Experiments.Resume, client.Experiments.Stop} { + if _, err := action(ctx, "proj", "exp1"); err != nil { + t.Fatal(err) + } + } + if err := client.Experiments.Delete(ctx, "proj", "exp1"); err != nil { + t.Fatal(err) + } + want := []string{ + "POST /projects/proj/experiments", + "POST /projects/proj/experiments/exp1", + "POST /projects/proj/experiments/exp1/actions/start", + "POST /projects/proj/experiments/exp1/actions/pause", + "POST /projects/proj/experiments/exp1/actions/resume", + "POST /projects/proj/experiments/exp1/actions/stop", + "DELETE /projects/proj/experiments/exp1", + } + if fmt.Sprint(requests) != fmt.Sprint(want) { + t.Fatalf("requests = %v, want %v", requests, want) + } +} + func TestExperimentsReadRoutesAndFilters(t *testing.T) { requests := []string{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/api/paths_gen.go b/internal/api/paths_gen.go index bcba7e7e..4519d14b 100644 --- a/internal/api/paths_gen.go +++ b/internal/api/paths_gen.go @@ -126,6 +126,22 @@ func pathExperiment(projectID string, experimentID string) string { return encodePath("projects", projectID, "experiments", experimentID) } +func pathExperimentActionsPause(projectID string, experimentID string) string { + return encodePath("projects", projectID, "experiments", experimentID, "actions", "pause") +} + +func pathExperimentActionsResume(projectID string, experimentID string) string { + return encodePath("projects", projectID, "experiments", experimentID, "actions", "resume") +} + +func pathExperimentActionsStart(projectID string, experimentID string) string { + return encodePath("projects", projectID, "experiments", experimentID, "actions", "start") +} + +func pathExperimentActionsStop(projectID string, experimentID string) string { + return encodePath("projects", projectID, "experiments", experimentID, "actions", "stop") +} + func pathExperimentResults(projectID string, experimentID string) string { return encodePath("projects", projectID, "experiments", experimentID, "results") } diff --git a/internal/cli/config_schema.go b/internal/cli/config_schema.go new file mode 100644 index 00000000..2aa0fa22 --- /dev/null +++ b/internal/cli/config_schema.go @@ -0,0 +1,111 @@ +package cli + +import "github.com/spf13/cobra" + +func configFieldsFor(cmd *cobra.Command) map[string]any { + switch commandPath(cmd) { + case "rc experiments create": + return experimentConfigFields(true) + case "rc experiments update": + return experimentConfigFields(false) + } + return nil +} + +func configField(kind, description string) map[string]any { + return map[string]any{"type": kind, "description": description} +} + +func configEnum(description string, values ...string) map[string]any { + return map[string]any{"type": "string", "description": description, "enum": values} +} + +func experimentConfigFields(create bool) map[string]any { + metricNames := []string{ + "initial_conversions", "initial_conversion_rate", "trials_started", "trials_completed", + "trials_converted", "trial_conversion_rate", "paid_customers", "conversion_to_paying", + "active_subscribers", "churned_subscribers", "refunded_customers", "realized_ltv_revenue", + "realized_ltv_per_customer", "realized_ltv_per_paying_customer", "total_mrr", + "mrr_per_customer", "mrr_per_paying_customer", + } + placements := map[string]any{ + "type": "object", "description": "Fallback Offerings and per-placement overrides", + "properties": map[string]any{ + "fallback_offering_a_id": configField("string", "Control Offering ID"), + "fallback_offering_b_id": configField("string", "Treatment Offering ID"), + "fallback_offering_c_id": configField("string", "Variant C Offering ID"), + "fallback_offering_d_id": configField("string", "Variant D Offering ID"), + "placement_offerings": map[string]any{ + "type": "array", "items": map[string]any{"type": "object", "required": []string{"placement_identifier"}, "properties": map[string]any{ + "placement_identifier": configField("string", "Placement identifier, such as onboarding"), + "offering_a_id": configField("string", "Control Offering ID for this placement"), + "offering_b_id": configField("string", "Treatment Offering ID for this placement"), + "offering_c_id": configField("string", "Variant C Offering ID for this placement"), + "offering_d_id": configField("string", "Variant D Offering ID for this placement"), + }}, + }, + }, + } + var fallbackFields []map[string]any + for _, field := range []string{"fallback_offering_a_id", "fallback_offering_b_id", "fallback_offering_c_id", "fallback_offering_d_id"} { + fallbackFields = append(fallbackFields, map[string]any{"required": []string{field}, "properties": map[string]any{field: map[string]any{"type": "string"}}}) + } + placements["description"] = "Fallback Offerings require at least one placement_offerings entry; otherwise the API discards them. An empty placement list without fallbacks clears placement configuration." + placements["if"] = map[string]any{"anyOf": fallbackFields} + placements["then"] = map[string]any{"required": []string{"placement_offerings"}, "properties": map[string]any{"placement_offerings": map[string]any{"type": "array", "minItems": 1}}} + duration := map[string]any{"type": "object", "properties": map[string]any{ + "conversion_rate_percentage": configField("number", "Expected conversion rate, as a percentage"), + "daily_enrolled_customers": configField("integer", "Estimated customers enrolled per day"), + "minimum_detectable_effect_percentage": configField("integer", "Minimum detectable effect, as a percentage"), + "chance_to_win_percentage": configField("integer", "Required confidence level, as a percentage"), + "conversion_rate_percentage_is_override": configField("boolean", "Conversion rate was manually overridden"), + "daily_enrolled_customers_is_override": configField("boolean", "Daily enrolled count was manually overridden"), + }} + secondaryNames := append(append([]string{}, metricNames...), "exposed_customers") + audience := map[string]any{ + "type": []string{"string", "null"}, + "description": "Audience ID; mutually exclusive with targeting_conditions. Omit or set null for all eligible customers when targeting_conditions is absent.", + } + fields := map[string]any{ + "display_name": configField("string", "Experiment name"), + "enrollment_percentage": map[string]any{"type": "integer", "minimum": 1, "maximum": 100, "description": "Percentage of eligible customers to enroll"}, + "offering_a_id": configField("string", "Control Offering ID"), + "offering_b_id": configField("string", "Treatment Offering ID"), + "offering_c_id": configField("string", "Optional third variant Offering ID"), + "offering_d_id": configField("string", "Optional fourth variant Offering ID"), + "audience_id": audience, + "targeting_conditions": targetingConditionsSchema(), + "placements": placements, + "notes": configField("string", "Experiment notes"), + "experiment_type": configEnum("Experiment type", "introductory_offer", "free_trial_offer", "paywall_design", "price_point", "subscription_duration", "subscription_ordering", "other"), + "primary_metric": configEnum("Primary metric", metricNames...), + "secondary_metrics": map[string]any{"type": "array", "items": configEnum("Secondary metric", secondaryNames...)}, + "enrollment_mode": configEnum("Enrollment mode", "only_new", "new_and_existing"), + "experiment_duration_settings": duration, + } + if create { + return map[string]any{"type": "object", "properties": fields, "required": []string{"display_name", "enrollment_percentage", "offering_a_id", "offering_b_id"}} + } + audience["description"] = "Omit to keep current targeting unless targeting_conditions is supplied. Set null to clear the audience and conditions, or supply targeting_conditions to replace them. A string selects an audience and cannot be combined with targeting_conditions." + audience["examples"] = []any{"aud_123", nil} + return map[string]any{"type": "object", "properties": fields, "description": "Partial update. Running experiments accept only enrollment_percentage; paused experiments cannot be edited."} +} + +func targetingConditionsSchema() map[string]any { + return map[string]any{ + "type": "array", "description": "Mutually exclusive with audience_id. Conditions are combined as an audience filter.", + "items": map[string]any{ + "type": "object", "required": []string{"field", "operator", "value"}, + "properties": map[string]any{ + "field": configEnum("Target field", "app_config", "app_version", "country", "custom_attribute", "platform", "sdk_version"), + "operator": configEnum("Use in/not in for arrays; =, !=, >, >=, <, <= for versions", "in", "not in", "=", "!=", ">", ">=", "<", "<="), + "value": map[string]any{"type": []string{"string", "integer", "array"}, "items": map[string]any{"type": []string{"string", "integer"}}, "description": "Version string for app_version/sdk_version; array for country/platform/app_config; strings or integers for custom attributes. See field_rules for each field."}, + "context": configField("string", "Required app ID for app_version; SDK flavor for sdk_version; attribute key for custom_attribute. Omit for other fields."), + }, + "examples": []map[string]any{ + {"field": "platform", "operator": "in", "value": []string{"ios"}}, + {"field": "app_version", "operator": ">=", "value": "1.2.0", "context": "app1a2b3c4"}, + }, + }, + } +} diff --git a/internal/cli/experiment_config.go b/internal/cli/experiment_config.go new file mode 100644 index 00000000..b9e7d3c5 --- /dev/null +++ b/internal/cli/experiment_config.go @@ -0,0 +1,46 @@ +package cli + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" +) + +const maxExperimentConfigBytes = 1 << 20 + +func readJSONConfig(path string, out any) error { + var reader io.Reader + if path == "-" { + reader = os.Stdin + } else { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + reader = file + } + data, err := io.ReadAll(io.LimitReader(reader, maxExperimentConfigBytes+1)) + if err != nil { + return err + } + if len(data) > maxExperimentConfigBytes { + return fmt.Errorf("config exceeds 1 MiB") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(out); err != nil { + return fmt.Errorf("invalid config JSON: %w", err) + } + if decoder.Decode(new(any)) != io.EOF { + return fmt.Errorf("config must contain one JSON object") + } + return nil +} + +func compactJSON(v any) string { + data, _ := json.Marshal(v) + return string(data) +} diff --git a/internal/cli/experiment_format.go b/internal/cli/experiment_format.go index 95390cb4..af94572e 100644 --- a/internal/cli/experiment_format.go +++ b/internal/cli/experiment_format.go @@ -108,15 +108,7 @@ func renderExperimentShow(rt *Runtime, experiment *api.Experiment) error { {Heading: "Timeline", Lines: timeline}, } if len(experiment.Conflicts) > 0 { - conflicts := make([]output.CardLine, 0, len(experiment.Conflicts)) - for _, conflict := range experiment.Conflicts { - name := conflict.DisplayName - if name == "" { - name = conflict.ID - } - conflicts = append(conflicts, output.CardLine{Key: conflict.ID, Value: name}) - } - sections = append(sections, output.CardSection{Heading: "Conflicts", Lines: conflicts}) + sections = append(sections, output.CardSection{Heading: "Conflicts", Lines: []output.CardLine{{Key: "Experiments", Value: experimentConflictsSummary(experiment.Conflicts)}}}) } if err := rt.Out.RenderCard(output.Card{ Title: experiment.DisplayName, @@ -240,6 +232,8 @@ func experimentConflictsSummary(conflicts []api.ExperimentConflict) string { parts[i] = conflict.DisplayName if parts[i] == "" { parts[i] = conflict.ID + } else if conflict.ID != "" { + parts[i] += " (" + conflict.ID + ")" } } return strings.Join(parts, ", ") diff --git a/internal/cli/experiment_placements_test.go b/internal/cli/experiment_placements_test.go new file mode 100644 index 00000000..f63aa301 --- /dev/null +++ b/internal/cli/experiment_placements_test.go @@ -0,0 +1,20 @@ +package cli + +import ( + "encoding/json" + "testing" +) + +func TestExperimentPlacementsAllowClearingAndConfiguredOverrides(t *testing.T) { + for _, raw := range []string{ + `null`, `{}`, `{"placement_offerings":[]}`, + `{"fallback_offering_a_id":null,"placement_offerings":[]}`, + `{"fallback_offering_a_id":"ofrng_a","placement_offerings":[{"placement_identifier":"onboarding","offering_a_id":"ofrng_a"}]}`, + } { + t.Run(raw, func(t *testing.T) { + if err := validExperimentPlacements(json.RawMessage(raw)); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/cli/experiment_update_format.go b/internal/cli/experiment_update_format.go new file mode 100644 index 00000000..14d73f1f --- /dev/null +++ b/internal/cli/experiment_update_format.go @@ -0,0 +1,51 @@ +package cli + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/revenuecat/cli/internal/api" +) + +func showExperimentUpdateChanges(rt *Runtime, changes api.ExperimentUpdate) { + keys := make([]string, 0, len(changes)) + for key := range changes { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + var value any + _ = json.Unmarshal(changes[key], &value) + rt.Out.Field("Change "+strings.ReplaceAll(key, "_", " "), experimentChangeValue(value)) + } +} + +func experimentChangeValue(value any) string { + switch value := value.(type) { + case nil: + return "clear" + case string: + return value + case []any: + parts := make([]string, len(value)) + for i, item := range value { + parts[i] = experimentChangeValue(item) + } + return strings.Join(parts, "; ") + case map[string]any: + keys := make([]string, 0, len(value)) + for key := range value { + keys = append(keys, key) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, strings.ReplaceAll(key, "_", " ")+": "+experimentChangeValue(value[key])) + } + return strings.Join(parts, " · ") + default: + return fmt.Sprint(value) + } +} diff --git a/internal/cli/experiments.go b/internal/cli/experiments.go index b0ca8a1f..828b380f 100644 --- a/internal/cli/experiments.go +++ b/internal/cli/experiments.go @@ -20,7 +20,11 @@ func newExperimentsCmd() *cobra.Command { Short: "Run and analyze offering experiments", Long: "Inspect experiments, compare their results, and manage their lifecycle.", } - cmd.AddCommand(newExperimentsListCmd(), newExperimentsShowCmd(), newExperimentsResultsCmd()) + cmd.AddCommand( + newExperimentsListCmd(), newExperimentsShowCmd(), newExperimentsResultsCmd(), + newExperimentsCreateCmd(), newExperimentsDuplicateCmd(), newExperimentsUpdateCmd(), newExperimentsDeleteCmd(), + newExperimentsStartCmd(), newExperimentsPauseCmd(), newExperimentsResumeCmd(), newExperimentsStopCmd(), + ) return cmd } diff --git a/internal/cli/experiments_duplicate.go b/internal/cli/experiments_duplicate.go new file mode 100644 index 00000000..12265830 --- /dev/null +++ b/internal/cli/experiments_duplicate.go @@ -0,0 +1,130 @@ +package cli + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + + "github.com/revenuecat/cli/internal/api" +) + +func newExperimentsDuplicateCmd() *cobra.Command { + var name string + cmd := &cobra.Command{ + Use: "duplicate [id]", + Short: "Copy an experiment into a new draft", + Long: "Copies an experiment's configuration into a new draft. Results, status, and timeline are not copied. The new draft does not enroll customers.", + Example: " rc experiments duplicate exp123 --name 'New paywall test'", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + id, err := requireID(rt, argAt(args, 0), "experiment", func() ([]PickerItem, error) { + return experimentPickerItems(cmd, client, projectID) + }) + if err != nil { + return err + } + source, err := client.Experiments.Get(cmd.Context(), projectID, id) + if err != nil { + return err + } + body, err := duplicateExperimentConfig(source) + if err != nil { + return err + } + if cmd.Flags().Changed("name") { + if name == "" { + return fmt.Errorf("--name cannot be empty") + } + body.DisplayName = name + } + copy, err := client.Experiments.Create(cmd.Context(), projectID, body) + if err != nil { + return err + } + rt.Out.Success("Created draft experiment " + copy.ID) + rt.Out.Hint("rc experiments start " + copy.ID) + return renderExperimentShow(rt, copy) + }, + } + cmd.Flags().StringVar(&name, "name", "", "name for the new draft (default: Copy of )") + return cmd +} + +func duplicateExperimentConfig(source *api.Experiment) (api.ExperimentCreate, error) { + if source.OfferingA == nil || source.OfferingB == nil || source.EnrollmentPercent == nil { + return api.ExperimentCreate{}, fmt.Errorf("experiment %s lacks configuration needed to duplicate it", source.ID) + } + body := api.ExperimentCreate{ + DisplayName: "Copy of " + source.DisplayName, + EnrollmentPercentage: *source.EnrollmentPercent, + OfferingAID: source.OfferingA.ID, + OfferingBID: source.OfferingB.ID, + Notes: source.Notes, + ExperimentType: source.ExperimentType, + PrimaryMetric: source.PrimaryMetric, + SecondaryMetrics: source.SecondaryMetrics, + EnrollmentMode: source.EnrollmentMode, + AudienceID: source.AudienceID, + } + if source.OfferingC != nil { + body.OfferingCID = &source.OfferingC.ID + } + if source.OfferingD != nil { + body.OfferingDID = &source.OfferingD.ID + } + if len(source.TargetingConditions) > 0 && source.AudienceID == nil { + body.TargetingConditions, _ = json.Marshal(source.TargetingConditions) + } + if source.DurationSettings != nil { + body.ExperimentDurationSettings, _ = json.Marshal(source.DurationSettings) + } + if source.Placements != nil { + placement := map[string]any{} + for _, variant := range []struct { + key string + offering *api.ExperimentOffering + }{ + {"fallback_offering_a_id", source.Placements.FallbackOfferingA}, + {"fallback_offering_b_id", source.Placements.FallbackOfferingB}, + {"fallback_offering_c_id", source.Placements.FallbackOfferingC}, + {"fallback_offering_d_id", source.Placements.FallbackOfferingD}, + } { + if variant.offering != nil { + placement[variant.key] = variant.offering.ID + } + } + rows := make([]map[string]string, 0, len(source.Placements.PlacementOfferings)) + for _, item := range source.Placements.PlacementOfferings { + row := map[string]string{"placement_identifier": item.PlacementIdentifier} + for _, variant := range []struct { + key string + offering *api.ExperimentOffering + }{ + {"offering_a_id", item.OfferingA}, + {"offering_b_id", item.OfferingB}, + {"offering_c_id", item.OfferingC}, + {"offering_d_id", item.OfferingD}, + } { + if variant.offering != nil { + row[variant.key] = variant.offering.ID + } + } + rows = append(rows, row) + } + if len(rows) > 0 { + placement["placement_offerings"] = rows + } + body.Placements, _ = json.Marshal(placement) + } + return body, nil +} diff --git a/internal/cli/experiments_lifecycle.go b/internal/cli/experiments_lifecycle.go new file mode 100644 index 00000000..b02a5efc --- /dev/null +++ b/internal/cli/experiments_lifecycle.go @@ -0,0 +1,534 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/charmbracelet/huh" + "github.com/spf13/cobra" + + "github.com/revenuecat/cli/internal/api" + "github.com/revenuecat/cli/internal/output" + "github.com/revenuecat/cli/internal/tui" +) + +func newExperimentsCreateCmd() *cobra.Command { + var name, control, treatment, config string + var enrollment int + cmd := &cobra.Command{ + Use: "create", + Short: "Create a draft experiment", + Long: "Creates a draft comparing two Offerings. Use --config for audience_id or targeting_conditions, placements, offering_c_id/offering_d_id, primary_metric, secondary_metrics, enrollment_mode, and experiment_duration_settings. Run rc schema experiments create for config fields and accepted values. Creating a draft does not enroll customers.", + Example: ` rc experiments create --name "New paywall" --control ofrng_a --treatment ofrng_b --enrollment 50 + echo '{"display_name":"New paywall","offering_a_id":"ofrng_a","offering_b_id":"ofrng_b","enrollment_percentage":50,"audience_id":"aud_123","primary_metric":"initial_conversion_rate"}' | rc experiments create --config - --json --no-input`, + RunE: func(cmd *cobra.Command, _ []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + body := api.ExperimentCreate{} + if config != "" { + if err := readJSONConfig(config, &body); err != nil { + return err + } + } + if cmd.Flags().Changed("name") { + body.DisplayName = name + } + if cmd.Flags().Changed("control") { + body.OfferingAID = control + } + if cmd.Flags().Changed("treatment") { + body.OfferingBID = treatment + } + if cmd.Flags().Changed("enrollment") { + body.EnrollmentPercentage = enrollment + } + if !rt.CanPrompt() { + var missing []string + if body.DisplayName == "" { + missing = append(missing, "--name") + } + if body.OfferingAID == "" { + missing = append(missing, "--control") + } + if body.OfferingBID == "" { + missing = append(missing, "--treatment") + } + if body.EnrollmentPercentage == 0 { + missing = append(missing, "--enrollment") + } + if len(missing) > 0 { + return fmt.Errorf("experiment input required: pass %s or --config ", strings.Join(missing, ", ")) + } + } else { + form := tui.Form(false) + if body.DisplayName == "" { + form.Field(huh.NewInput().Title("Experiment name").Value(&body.DisplayName).Validate(tui.Required("name"))) + } + if body.EnrollmentPercentage == 0 { + percent := "50" + form.Field(huh.NewInput().Title("Enrollment percentage (1–100)").Value(&percent)) + if err := form.Run(); err != nil { + return err + } + body.EnrollmentPercentage, err = strconv.Atoi(percent) + if err != nil { + return fmt.Errorf("enrollment must be an integer from 1 to 100") + } + } else if err := form.Run(); err != nil { + return err + } + if body.OfferingAID == "" || body.OfferingBID == "" { + client, err := rt.API() + if err != nil { + return err + } + fetch := func() ([]PickerItem, error) { + return offeringPickerItems(cmd.Context(), client, projectID) + } + if body.OfferingAID == "" { + body.OfferingAID, err = requireID(rt, "", "control offering", fetch) + if err != nil { + return err + } + } + if body.OfferingBID == "" { + body.OfferingBID, err = requireID(rt, "", "treatment offering", fetch) + if err != nil { + return err + } + } + } + } + if body.OfferingAID == body.OfferingBID { + return fmt.Errorf("control and treatment must use different Offerings") + } + if body.EnrollmentPercentage < 1 || body.EnrollmentPercentage > 100 { + return fmt.Errorf("enrollment must be an integer from 1 to 100") + } + if body.AudienceID != nil && len(body.TargetingConditions) > 0 { + var conditions []json.RawMessage + if err := json.Unmarshal(body.TargetingConditions, &conditions); err != nil { + return fmt.Errorf("targeting_conditions must be an array or null") + } + if len(conditions) > 0 { + return fmt.Errorf("audience_id and targeting_conditions cannot both be set") + } + } + client, err := rt.API() + if err != nil { + return err + } + if err := validExperimentPlacements(body.Placements); err != nil { + return err + } + experiment, err := client.Experiments.Create(cmd.Context(), projectID, body) + if err != nil { + return err + } + rt.Out.Success(fmt.Sprintf("Created draft experiment %s", experiment.ID)) + rt.Out.Hint("rc experiments start " + experiment.ID) + return renderExperimentShow(rt, experiment) + }, + } + cmd.Flags().StringVar(&name, "name", "", "experiment display name") + cmd.Flags().StringVar(&control, "control", "", "control Offering ID") + cmd.Flags().StringVar(&treatment, "treatment", "", "treatment Offering ID") + cmd.Flags().IntVar(&enrollment, "enrollment", 0, "percentage of eligible customers to enroll (1–100)") + cmd.Flags().StringVar(&config, "config", "", "JSON config file; use - for stdin") + return cmd +} + +func newExperimentsUpdateCmd() *cobra.Command { + var config string + cmd := &cobra.Command{ + Use: "update [id]", + Short: "Update an experiment", + Long: "Partially updates an experiment from a JSON object. Only supplied fields change. Running experiments accept only enrollment_percentage and require confirmation. Paused experiments cannot be edited. Run rc schema experiments update to inspect config fields.", + Example: ` echo '{"display_name":"Updated test"}' | rc experiments update exp123 --config - --no-input`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if config == "" { + return fmt.Errorf("pass --config or --config - for stdin") + } + body := api.ExperimentUpdate{} + if err := readJSONConfig(config, &body); err != nil { + return err + } + if len(body) == 0 { + return fmt.Errorf("config must contain at least one field") + } + if err := validExperimentUpdate(body); err != nil { + return err + } + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + id, err := requireID(rt, argAt(args, 0), "experiment", func() ([]PickerItem, error) { + return experimentPickerItems(cmd, client, projectID) + }) + if err != nil { + return err + } + current, err := client.Experiments.Get(cmd.Context(), projectID, id) + if err != nil { + return err + } + if current.Status != "draft" && current.Status != "running" { + return fmt.Errorf("experiment %s is %s; only drafts and running experiments can be updated", id, current.Status) + } + if current.Status == "running" { + for field := range body { + if field != "enrollment_percentage" { + return fmt.Errorf("running experiments accept only enrollment_percentage; remove %q", field) + } + } + showExperimentEnrollmentState(rt, current, "Review the current experiment before changing enrollment.") + showExperimentUpdateChanges(rt, body) + rt.Out.Notice("Changes to a running experiment may affect customers being enrolled now.") + rt.Out.Plan([]string{"Update the running experiment"}) + if err := confirmOrAbort(rt, "Update running experiment now?"); err != nil { + return err + } + } + experiment, err := client.Experiments.Update(cmd.Context(), projectID, id, body) + if err != nil { + return err + } + rt.Out.Success("Updated experiment " + experiment.ID) + return renderExperimentShow(rt, experiment) + }, + } + cmd.Flags().StringVar(&config, "config", "", "JSON object of fields to update; use - for stdin") + return cmd +} + +func newExperimentsStartCmd() *cobra.Command { + return &cobra.Command{ + Use: "start [id]", + Short: "Start a draft experiment", + Long: "Starts enrollment for the experiment after showing its current configuration. Requires confirmation or --yes.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + id, err := requireID(rt, argAt(args, 0), "experiment", func() ([]PickerItem, error) { + return experimentPickerItems(cmd, client, projectID) + }) + if err != nil { + return err + } + current, err := client.Experiments.Get(cmd.Context(), projectID, id) + if err != nil { + return err + } + if current.Status != "draft" { + return fmt.Errorf("experiment %s is %s; only drafts can be started", id, current.Status) + } + showExperimentEnrollmentState(rt, current, "Start enrolling eligible customers in the experiment.") + rt.Out.Plan([]string{"Start the experiment and enroll eligible customers"}) + if err := confirmOrAbort(rt, "Start experiment now?"); err != nil { + return err + } + rt.Out.Info("Starting experiment…") + experiment, err := client.Experiments.Start(cmd.Context(), projectID, id) + if err != nil { + return err + } + rt.Out.Success("Started experiment " + experiment.ID) + if err := rt.Out.RenderCard(output.Card{ + Title: experiment.DisplayName, + Sections: []output.CardSection{{Lines: []output.CardLine{{Key: "ID", Value: experiment.ID}, {Key: "Status", Value: experiment.Status}}}}, + Raw: experiment, + }); err != nil { + return err + } + rt.Out.Hint("rc experiments results " + experiment.ID) + return nil + }, + } +} + +func showExperimentEnrollmentState(rt *Runtime, current *api.Experiment, lead string) { + rt.Out.Title("Experiment — " + current.DisplayName) + rt.Out.Lead(lead) + rt.Out.Field("Status", current.Status) + if current.OfferingA != nil { + rt.Out.Field("Control", experimentOfferingLabel(current.OfferingA)) + } + if current.OfferingB != nil { + rt.Out.Field("Treatment", experimentOfferingLabel(current.OfferingB)) + } + if current.OfferingC != nil { + rt.Out.Field("Treatment C", experimentOfferingLabel(current.OfferingC)) + } + if current.OfferingD != nil { + rt.Out.Field("Treatment D", experimentOfferingLabel(current.OfferingD)) + } + if current.EnrollmentPercent != nil { + rt.Out.Field("Enrollment", fmt.Sprintf("%d%%", *current.EnrollmentPercent)) + } + if current.AudienceID != nil { + rt.Out.Field("Audience", *current.AudienceID) + } else if len(current.TargetingConditions) > 0 { + for i, condition := range current.TargetingConditions { + rt.Out.Field(fmt.Sprintf("Condition %d", i+1), experimentConditionLabel(condition)) + } + } else { + rt.Out.Field("Audience", "All eligible customers") + } + for _, placement := range experimentPlacementLines(current.Placements) { + rt.Out.Field("Placement "+placement.Key, placement.Value) + } + if current.ExperimentType != nil { + rt.Out.Field("Type", *current.ExperimentType) + } + if current.PrimaryMetric != nil { + rt.Out.Field("Primary metric", *current.PrimaryMetric) + } + if len(current.SecondaryMetrics) > 0 { + rt.Out.Field("Secondary metrics", strings.Join(current.SecondaryMetrics, ", ")) + } + if current.EnrollmentMode != nil { + rt.Out.Field("Enrollment mode", *current.EnrollmentMode) + } + if current.DurationSettings != nil { + rt.Out.Field("Duration", experimentDurationSummary(current.DurationSettings)) + } + if current.Priority != nil { + rt.Out.Field("Priority", strconv.Itoa(*current.Priority)) + } + if len(current.Conflicts) > 0 { + rt.Out.Notice("Experiment conflicts: " + experimentConflictsSummary(current.Conflicts)) + } +} + +func newExperimentsPauseCmd() *cobra.Command { + return &cobra.Command{ + Use: "pause [id]", Short: "Pause an experiment", Long: "Pauses enrollment after showing the current experiment configuration. Requires confirmation or --yes.", Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + id, err := requireID(rt, argAt(args, 0), "experiment", func() ([]PickerItem, error) { + return experimentPickerItems(cmd, client, projectID) + }) + if err != nil { + return err + } + current, err := client.Experiments.Get(cmd.Context(), projectID, id) + if err != nil { + return err + } + if current.Status != "running" { + return fmt.Errorf("experiment %s is %s; only running experiments can be paused", id, current.Status) + } + showExperimentEnrollmentState(rt, current, "Pause enrollment in the experiment.") + rt.Out.Plan([]string{"Pause the experiment and stop enrolling customers"}) + if err := confirmOrAbort(rt, "Pause experiment now?"); err != nil { + return err + } + rt.Out.Info("Pausing experiment…") + experiment, err := client.Experiments.Pause(cmd.Context(), projectID, id) + if err != nil { + return err + } + rt.Out.Success("Paused experiment " + experiment.ID) + return renderExperimentShow(rt, experiment) + }, + } +} + +func newExperimentsResumeCmd() *cobra.Command { + return &cobra.Command{ + Use: "resume [id]", Short: "Resume enrollment in a paused experiment", Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + id, err := requireID(rt, argAt(args, 0), "experiment", func() ([]PickerItem, error) { + return experimentPickerItems(cmd, client, projectID) + }) + if err != nil { + return err + } + current, err := client.Experiments.Get(cmd.Context(), projectID, id) + if err != nil { + return err + } + if current.Status != "paused" { + return fmt.Errorf("experiment %s is %s; only paused experiments can be resumed", id, current.Status) + } + showExperimentEnrollmentState(rt, current, "Resume enrolling eligible customers in the experiment.") + rt.Out.Plan([]string{"Resume the experiment and enroll eligible customers"}) + if err := confirmOrAbort(rt, "Resume experiment now?"); err != nil { + return err + } + rt.Out.Info("Resuming experiment…") + experiment, err := client.Experiments.Resume(cmd.Context(), projectID, id) + if err != nil { + return err + } + rt.Out.Success("Resumed experiment " + experiment.ID) + return renderExperimentShow(rt, experiment) + }, + } +} + +func newExperimentsStopCmd() *cobra.Command { + return &cobra.Command{ + Use: "stop [id]", Short: "Permanently stop an experiment", Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + id, err := requireID(rt, argAt(args, 0), "experiment", func() ([]PickerItem, error) { + return experimentPickerItems(cmd, client, projectID) + }) + if err != nil { + return err + } + current, err := client.Experiments.Get(cmd.Context(), projectID, id) + if err != nil { + return err + } + showExperimentEnrollmentState(rt, current, "Permanently stop the experiment.") + rt.Out.Plan([]string{"Stop the experiment permanently"}) + if err := confirmOrAbort(rt, "Permanently stop experiment "+output.SanitizeLine(current.DisplayName)+" ("+output.SanitizeLine(id)+")?"); err != nil { + return err + } + experiment, err := client.Experiments.Stop(cmd.Context(), projectID, id) + if err != nil { + return err + } + rt.Out.Success("Stopped experiment " + experiment.ID) + return renderExperimentShow(rt, experiment) + }, + } +} + +func newExperimentsDeleteCmd() *cobra.Command { + return &cobra.Command{ + Use: "delete [id]", + Short: "Delete a draft experiment", + Long: "Permanently deletes a draft experiment. Requires confirmation or --yes.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + id, err := requireID(rt, argAt(args, 0), "experiment", func() ([]PickerItem, error) { + return experimentPickerItems(cmd, client, projectID) + }) + if err != nil { + return err + } + current, err := client.Experiments.Get(cmd.Context(), projectID, id) + if err != nil { + return err + } + if current.Status != "draft" { + return fmt.Errorf("experiment %s is %s; only draft experiments can be deleted", id, current.Status) + } + if err := confirmOrAbort(rt, "Delete draft experiment "+output.SanitizeLine(current.DisplayName)+" ("+output.SanitizeLine(id)+")?"); err != nil { + return err + } + if err := client.Experiments.Delete(cmd.Context(), projectID, id); err != nil { + return err + } + rt.Out.Success("Deleted experiment " + id) + return rt.Out.Render(map[string]any{"id": id, "deleted": true}) + }, + } +} + +func validExperimentUpdate(body api.ExperimentUpdate) error { + allowed := map[string]bool{ + "display_name": true, "enrollment_percentage": true, "offering_a_id": true, + "offering_b_id": true, "offering_c_id": true, "offering_d_id": true, + "targeting_conditions": true, "audience_id": true, "placements": true, + "notes": true, "experiment_type": true, "primary_metric": true, + "secondary_metrics": true, "enrollment_mode": true, "experiment_duration_settings": true, + } + for key, value := range body { + if !allowed[key] { + return fmt.Errorf("unknown experiment field %q", key) + } + if !json.Valid(value) { + return fmt.Errorf("invalid JSON value for %q", key) + } + if key == "placements" { + if err := validExperimentPlacements(value); err != nil { + return err + } + } + if key == "enrollment_percentage" { + var percentage int + if err := json.Unmarshal(value, &percentage); err != nil || percentage < 1 || percentage > 100 { + return fmt.Errorf("enrollment must be an integer from 1 to 100") + } + } + } + return nil +} + +func validExperimentPlacements(raw json.RawMessage) error { + if len(raw) == 0 { + return nil + } + var placements struct { + FallbackA *string `json:"fallback_offering_a_id"` + FallbackB *string `json:"fallback_offering_b_id"` + FallbackC *string `json:"fallback_offering_c_id"` + FallbackD *string `json:"fallback_offering_d_id"` + Offerings []json.RawMessage `json:"placement_offerings"` + } + if err := json.Unmarshal(raw, &placements); err != nil { + return fmt.Errorf("placements must be an object or null") + } + if len(placements.Offerings) == 0 && (placements.FallbackA != nil || placements.FallbackB != nil || placements.FallbackC != nil || placements.FallbackD != nil) { + return fmt.Errorf("placements with fallback Offering IDs require at least one placement_offerings entry; the API otherwise discards the fallbacks") + } + return nil +} diff --git a/internal/cli/experiments_test.go b/internal/cli/experiments_test.go index 235b9699..c14c7ca6 100644 --- a/internal/cli/experiments_test.go +++ b/internal/cli/experiments_test.go @@ -2,9 +2,12 @@ package cli_test import ( "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" ) @@ -133,6 +136,304 @@ func TestExperimentsDiscoverable(t *testing.T) { } } +func TestExperimentStartRequiresApprovalBeforeMutation(t *testing.T) { + mutations := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodPost { + mutations++ + _, _ = io.WriteString(w, `{"object":"experiment","id":"exp1","display_name":"New paywall","status":"running","created_at":1,"updated_at":2}`) + return + } + _, _ = io.WriteString(w, `{"object":"experiment","id":"exp1","display_name":"New paywall","status":"draft","created_at":1,"updated_at":2,"enrollment_percentage":50,"offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"}}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + args := []string{"experiments", "start", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input"} + _, _, err := runAgentCmd(t, args...) + if err == nil || !strings.Contains(err.Error(), "--yes") || mutations != 0 { + t.Fatalf("expected approval error before mutation; err=%v mutations=%d", err, mutations) + } + args = append(args, "--yes", "--json") + out, stderr, err := runAgentCmd(t, args...) + if err != nil || stderr != "" || mutations != 1 || !strings.Contains(out, `"status": "running"`) { + t.Fatalf("approved start failed: err=%v stderr=%q mutations=%d out=%s", err, stderr, mutations, out) + } +} + +func TestExperimentResumeRequiresApprovalBeforeMutation(t *testing.T) { + mutations := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodPost { + mutations++ + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"New paywall","status":"running"}`) + return + } + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"New paywall","status":"paused","enrollment_percentage":50,"offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"}}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + args := []string{"experiments", "resume", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input"} + _, stderr, err := runAgentCmd(t, args...) + if err == nil || !strings.Contains(err.Error(), "--yes") || mutations != 0 || !strings.Contains(stderr, "Control") { + t.Fatalf("expected reviewed approval before resume; err=%v mutations=%d stderr=%q", err, mutations, stderr) + } + args = append(args, "--yes", "--json") + _, _, err = runAgentCmd(t, args...) + if err != nil || mutations != 1 { + t.Fatalf("approved resume failed: err=%v mutations=%d", err, mutations) + } +} + +func TestExperimentPauseShowsReadableResult(t *testing.T) { + mutations := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + status := "running" + if r.Method == http.MethodPost { + mutations++ + status = "paused" + } + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Paywall test","status":"`+status+`","offering_a":{"id":"ofrng_a","display_name":"Control"},"offering_b":{"id":"ofrng_b","display_name":"Treatment"},"targeting_conditions":[{"field":"platform","operator":"in","value":["ios"]}],"placements":{"placement_offerings":[{"placement_identifier":"onboarding","offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"}}]}}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + args := []string{"experiments", "pause", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input"} + _, preview, err := runAgentCmd(t, args...) + if err == nil || !strings.Contains(err.Error(), "--yes") || mutations != 0 || !strings.Contains(preview, "platform in ios") { + t.Fatalf("expected reviewed approval before pause: err=%v mutations=%d preview=%q", err, mutations, preview) + } + out, _, err := runAgentCmd(t, append(args, "--yes")...) + if err != nil { + t.Fatal(err) + } + if mutations != 1 { + t.Fatalf("approved pause made %d mutations, want 1", mutations) + } + for _, want := range []string{"ofrng_a (Control)", "platform in ios", "onboarding", "Treatment ofrng_b"} { + if !strings.Contains(out, want) { + t.Fatalf("missing %q from pause result: %s", want, out) + } + } +} + +func TestRunningExperimentUpdateShowsChangesBeforeApproval(t *testing.T) { + mutations := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodPost { + mutations++ + } + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"New paywall","status":"running","enrollment_percentage":50,"offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"}}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + configPath := filepath.Join(t.TempDir(), "changes.json") + if err := os.WriteFile(configPath, []byte(`{"enrollment_percentage":80}`), 0o600); err != nil { + t.Fatal(err) + } + _, stderr, err := runAgentCmd(t, "experiments", "update", "exp1", "--config", configPath, "--project-id", "proj", "--api-key", "sk_test", "--no-input") + if err == nil || !strings.Contains(err.Error(), "--yes") || mutations != 0 || !strings.Contains(stderr, "Change enrollment percentage") || !strings.Contains(stderr, "80") { + t.Fatalf("expected reviewed approval before update; err=%v mutations=%d stderr=%q", err, mutations, stderr) + } +} + +func TestExperimentUpdateRejectsFieldsAfterStart(t *testing.T) { + for _, tc := range []struct { + name, status, config, want string + }{ + {"running name", "running", `{"display_name":"Changed"}`, "only enrollment_percentage"}, + {"running mixed fields", "running", `{"enrollment_percentage":80,"notes":"Changed"}`, "only enrollment_percentage"}, + {"paused enrollment", "paused", `{"enrollment_percentage":80}`, "only drafts and running experiments"}, + {"stopped enrollment", "stopped", `{"enrollment_percentage":80}`, "only drafts and running experiments"}, + } { + t.Run(tc.name, func(t *testing.T) { + mutations := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodPost { + mutations++ + } + _, _ = fmt.Fprintf(w, `{"id":"exp1","display_name":"Original","status":%q,"enrollment_percentage":50}`, tc.status) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + configPath := filepath.Join(t.TempDir(), "changes.json") + if err := os.WriteFile(configPath, []byte(tc.config), 0o600); err != nil { + t.Fatal(err) + } + _, _, err := runAgentCmd(t, "experiments", "update", "exp1", "--config", configPath, + "--project-id", "proj", "--api-key", "sk_test", "--yes", "--no-input") + if err == nil || !strings.Contains(err.Error(), tc.want) || mutations != 0 { + t.Fatalf("err=%v mutations=%d, want %q and no mutation", err, mutations, tc.want) + } + }) + } +} + +func TestExperimentCreateListsMissingFlags(t *testing.T) { + _, _, err := runCmd(t, "experiments", "create", "--project-id", "proj", "--api-key", "sk_test", "--no-input") + if err == nil { + t.Fatal("expected missing input error") + } + for _, flag := range []string{"--name", "--control", "--treatment", "--enrollment"} { + if !strings.Contains(err.Error(), flag) { + t.Fatalf("missing %s from error: %v", flag, err) + } + } +} + +func TestExperimentDuplicateCopiesConfigurationIntoDraft(t *testing.T) { + var posted map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method == http.MethodGet { + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Original","status":"running","enrollment_percentage":40,"offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"},"offering_c":{"id":"ofrng_c"},"targeting_conditions":[{"field":"platform","operator":"in","value":["ios"]}],"placements":{"fallback_offering_a":{"id":"ofrng_fa"},"placement_offerings":[{"placement_identifier":"onboarding","offering_a":{"id":"ofrng_pa"},"offering_b":{"id":"ofrng_pb"}}]},"experiment_duration_settings":{"chance_to_win_percentage":95},"primary_metric":"initial_conversion_rate","enrollment_mode":"only_new"}`) + return + } + if r.URL.Path != "/projects/proj/experiments" { + t.Errorf("unexpected mutation path: %s", r.URL.Path) + } + if err := json.NewDecoder(r.Body).Decode(&posted); err != nil { + t.Error(err) + } + _, _ = io.WriteString(w, `{"id":"exp2","display_name":"New test","status":"draft"}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + _, _, err := runAgentCmd(t, "experiments", "duplicate", "exp1", "--name", "New test", "--project-id", "proj", "--api-key", "sk_test", "--json", "--no-input") + if err != nil { + t.Fatal(err) + } + if posted["display_name"] != "New test" || posted["offering_c_id"] != "ofrng_c" || posted["primary_metric"] != "initial_conversion_rate" { + t.Fatalf("missing copied fields: %#v", posted) + } + placements := posted["placements"].(map[string]any) + rows := placements["placement_offerings"].([]any) + if placements["fallback_offering_a_id"] != "ofrng_fa" || rows[0].(map[string]any)["offering_b_id"] != "ofrng_pb" { + t.Fatalf("placement IDs not copied: %#v", placements) + } + if _, exists := posted["status"]; exists { + t.Fatalf("status should not be copied: %#v", posted) + } +} + +func TestExperimentStopAndDeleteRequireApproval(t *testing.T) { + for _, tc := range []struct { + command string + method string + path string + }{ + {"stop", http.MethodPost, "/projects/proj/experiments/exp1/actions/stop"}, + {"delete", http.MethodDelete, "/projects/proj/experiments/exp1"}, + } { + t.Run(tc.command, func(t *testing.T) { + mutations := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && r.URL.Path == "/projects/proj/experiments/exp1" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Test","status":"draft"}`) + return + } + if r.Method != tc.method || r.URL.Path != tc.path { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + } + mutations++ + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Test","status":"stopped"}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + args := []string{"experiments", tc.command, "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input"} + _, preview, err := runAgentCmd(t, args...) + if tc.command == "stop" && (!strings.Contains(preview, "Test") || !strings.Contains(preview, "Status") || !strings.Contains(preview, "draft")) { + t.Fatalf("missing stop preview: %s", preview) + } + if err == nil || !strings.Contains(err.Error(), "--yes") || mutations != 0 { + t.Fatalf("expected approval before mutation: err=%v mutations=%d", err, mutations) + } + _, _, err = runAgentCmd(t, append(args, "--yes", "--json")...) + if err != nil || mutations != 1 { + t.Fatalf("approved %s failed: err=%v mutations=%d", tc.command, err, mutations) + } + }) + } +} + +func TestExperimentDeleteRejectsNonDraft(t *testing.T) { + for _, status := range []string{"running", "paused", "stopped"} { + t.Run(status, func(t *testing.T) { + mutations := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + mutations++ + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Test","status":"`+status+`"}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + _, _, err := runAgentCmd(t, "experiments", "delete", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--yes", "--no-input") + if err == nil || !strings.Contains(err.Error(), "only draft experiments can be deleted") || mutations != 0 { + t.Fatalf("expected non-draft guard: err=%v mutations=%d", err, mutations) + } + }) + } +} + +func TestExperimentCreateAndApprovedUpdate(t *testing.T) { + var created, updated map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodPost && r.URL.Path == "/projects/proj/experiments": + if err := json.NewDecoder(r.Body).Decode(&created); err != nil { + t.Error(err) + } + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Test","status":"draft","enrollment_percentage":50,"offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"}}`) + case r.Method == http.MethodGet && r.URL.Path == "/projects/proj/experiments/exp1": + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Test","status":"running","enrollment_percentage":50,"offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"}}`) + case r.Method == http.MethodPost && r.URL.Path == "/projects/proj/experiments/exp1": + if err := json.NewDecoder(r.Body).Decode(&updated); err != nil { + t.Error(err) + } + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Test","status":"running","enrollment_percentage":60,"offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"}}`) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + } + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + _, _, err := runAgentCmd(t, "experiments", "create", "--name", "Test", "--control", "ofrng_a", "--treatment", "ofrng_b", "--enrollment", "50", "--project-id", "proj", "--api-key", "sk_test", "--json", "--no-input") + if err != nil || created["enrollment_percentage"] != float64(50) { + t.Fatalf("create failed: err=%v body=%v", err, created) + } + configPath := filepath.Join(t.TempDir(), "update.json") + if err := os.WriteFile(configPath, []byte(`{"enrollment_percentage":60}`), 0o600); err != nil { + t.Fatal(err) + } + _, _, err = runAgentCmd(t, "experiments", "update", "exp1", "--config", configPath, "--project-id", "proj", "--api-key", "sk_test", "--yes", "--json", "--no-input") + if err != nil || updated["enrollment_percentage"] != float64(60) { + t.Fatalf("approved update failed: err=%v body=%v", err, updated) + } +} + +func TestExperimentUpdateRejectsInvalidEnrollmentBeforeRequest(t *testing.T) { + for _, value := range []string{"0", "101", "null", `"5"`, "5.5"} { + t.Run(value, func(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "update.json") + if err := os.WriteFile(configPath, []byte(`{"enrollment_percentage":`+value+`}`), 0o600); err != nil { + t.Fatal(err) + } + _, _, err := runAgentCmd(t, "experiments", "update", "exp1", "--config", configPath, "--project-id", "proj", "--api-key", "sk_test", "--no-input") + if err == nil || !strings.Contains(err.Error(), "1 to 100") { + t.Fatalf("expected local enrollment validation, got %v", err) + } + }) + } +} + func TestExperimentResultsPreservesPercentageUnitsAndShowsFilters(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -223,6 +524,106 @@ func TestExperimentResultsIncludesFourVariants(t *testing.T) { } } +func TestExperimentUpdatePreservesOmittedAndNullAudience(t *testing.T) { + for _, tc := range []struct { + name, config string + present bool + value any + }{ + {"omitted", `{"notes":"Keep targeting"}`, false, nil}, + {"cleared", `{"audience_id":null}`, true, nil}, + {"replaced", `{"audience_id":"aud_new"}`, true, "aud_new"}, + } { + t.Run(tc.name, func(t *testing.T) { + var posted map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + if err := json.NewDecoder(r.Body).Decode(&posted); err != nil { + t.Error(err) + } + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Original","status":"draft","audience_id":"aud_original"}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + configPath := filepath.Join(t.TempDir(), "update.json") + if err := os.WriteFile(configPath, []byte(tc.config), 0o600); err != nil { + t.Fatal(err) + } + _, _, err := runAgentCmd(t, "experiments", "update", "exp1", "--config", configPath, "--project-id", "proj", "--api-key", "sk_test", "--json", "--no-input") + value, present := posted["audience_id"] + if err != nil || posted == nil || present != tc.present || value != tc.value { + t.Fatalf("err=%v posted=%v; audience present=%v value=%v", err, posted, present, value) + } + }) + } +} + +func TestExperimentUpdateSchemaExplainsAudienceOmissionAndNull(t *testing.T) { + out, _, err := runCmd(t, "schema", "experiments", "update", "--json") + if err != nil { + t.Fatal(err) + } + var envelope struct { + Data struct { + ConfigFields struct { + Properties struct { + Audience struct { + Type []string `json:"type"` + Description string `json:"description"` + } `json:"audience_id"` + } `json:"properties"` + } `json:"config_fields"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(out), &envelope); err != nil { + t.Fatal(err) + } + audience := envelope.Data.ConfigFields.Properties.Audience + if strings.Join(audience.Type, ",") != "string,null" || !strings.Contains(audience.Description, "Omit to keep") || !strings.Contains(audience.Description, "Set null to clear") { + t.Fatalf("audience schema does not explain both behaviors: %+v", audience) + } +} + +func TestExperimentCreateAudienceWithEmptyConditions(t *testing.T) { + for _, conditions := range []string{"null", "[]", `[{"field":"country","operator":"in","value":["US"]}]`} { + t.Run(conditions, func(t *testing.T) { + posted := false + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + posted = true + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"exp1","status":"draft"}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + path := filepath.Join(t.TempDir(), "create.json") + body := `{"display_name":"Test","offering_a_id":"a","offering_b_id":"b","enrollment_percentage":50,"audience_id":"aud1","targeting_conditions":` + conditions + `}` + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + _, _, err := runAgentCmd(t, "experiments", "create", "--config", path, "--project-id", "proj", "--api-key", "sk_test", "--json", "--no-input") + wantSuccess := conditions == "null" || conditions == "[]" + if (err == nil) != wantSuccess || posted != wantSuccess { + t.Fatalf("err=%v posted=%v wantSuccess=%v", err, posted, wantSuccess) + } + }) + } +} + +func TestExperimentDeleteSanitizesConfirmation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Test\u001b[31m\nforged","status":"draft"}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + _, _, err := runAgentCmd(t, "experiments", "delete", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input") + if err == nil || !strings.Contains(err.Error(), "Test") || !strings.Contains(err.Error(), "exp1") || strings.ContainsAny(err.Error(), "\x1b\n") { + t.Fatalf("unsafe confirmation: %v", err) + } +} + func TestExperimentShowFallsBackOnlyOnExpandedForbidden(t *testing.T) { for _, tc := range []struct { name string @@ -285,3 +686,38 @@ func TestExperimentShowFallsBackOnlyOnExpandedForbidden(t *testing.T) { }) } } + +func TestExperimentPlacementsRejectIgnoredFallbacksBeforeMutation(t *testing.T) { + for _, command := range []string{"create", "update"} { + for _, placement := range []string{`{"fallback_offering_a_id":"a"}`, `{"fallback_offering_b_id":"b","placement_offerings":[]}`, `{"fallback_offering_c_id":"c","placement_offerings":null}`, `{"fallback_offering_d_id":"d","placement_offerings":[]}`} { + t.Run(command+placement, func(t *testing.T) { + mutations := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + mutations++ + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"exp1","status":"draft"}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + config := `{"placements":` + placement + `}` + args := []string{"experiments", command} + if command == "create" { + args = append(args, "--name", "Test", "--control", "a", "--treatment", "b", "--enrollment", "50") + } else { + args = append(args, "exp1") + } + path := filepath.Join(t.TempDir(), "placements.json") + if err := os.WriteFile(path, []byte(config), 0600); err != nil { + t.Fatal(err) + } + args = append(args, "--config", path, "--project-id", "proj", "--api-key", "sk_test", "--no-input", "--yes") + _, _, err := runAgentCmd(t, args...) + if err == nil || !strings.Contains(err.Error(), "at least one placement_offerings") || mutations != 0 { + t.Fatalf("err=%v mutations=%d", err, mutations) + } + }) + } + } +} diff --git a/internal/cli/schema.go b/internal/cli/schema.go index a88c52b0..43a05bd2 100644 --- a/internal/cli/schema.go +++ b/internal/cli/schema.go @@ -135,6 +135,9 @@ func commandSchema(c *cobra.Command) map[string]any { if isExperimental(c) { schema["experimental"] = true } + if fields := configFieldsFor(c); fields != nil { + schema["config_fields"] = fields + } addHumanRequirement(schema, c) return schema } diff --git a/internal/cli/schema_test.go b/internal/cli/schema_test.go index f5b9da84..bb769e39 100644 --- a/internal/cli/schema_test.go +++ b/internal/cli/schema_test.go @@ -1,12 +1,28 @@ package cli import ( + "encoding/json" "strings" "testing" "github.com/spf13/cobra" ) +func TestExperimentConfigSchemaExplainsNestedFields(t *testing.T) { + root := NewRootCmd("test") + for _, path := range []string{"experiments create", "experiments update"} { + data, err := json.Marshal(commandSchema(findCommand(t, root, path))["config_fields"]) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"primary_metric", "new_and_existing", "experiment_duration_settings", "placement_offerings", "app_version", "context"} { + if !strings.Contains(string(data), want) { + t.Errorf("%s schema missing %q", path, want) + } + } + } +} + func findCommand(t *testing.T, root *cobra.Command, path string) *cobra.Command { t.Helper() cur := root @@ -135,3 +151,31 @@ func hasRunnableDescendant(c *cobra.Command) bool { } return false } + +func TestTargetingConditionSchemaUsesStandardValueTypes(t *testing.T) { + value := targetingConditionsSchema()["items"].(map[string]any)["properties"].(map[string]any)["value"].(map[string]any) + types, ok := value["type"].([]string) + if !ok || len(types) != 3 || !contains(types, "string") || !contains(types, "integer") || !contains(types, "array") { + t.Fatalf("invalid value types: %v", value) + } + items := value["items"].(map[string]any)["type"].([]string) + if len(items) != 2 || !contains(items, "string") || !contains(items, "integer") { + t.Fatalf("invalid array item types: %v", items) + } +} + +func TestExperimentPlacementSchemaRequiresOverridesForFallbacks(t *testing.T) { + for _, create := range []bool{true, false} { + placements := experimentConfigFields(create)["properties"].(map[string]any)["placements"].(map[string]any) + cases := placements["if"].(map[string]any)["anyOf"].([]map[string]any) + if len(cases) != 4 { + t.Fatalf("missing fallback variants: %v", cases) + } + then := placements["then"].(map[string]any) + required := then["required"].([]string) + list := then["properties"].(map[string]any)["placement_offerings"].(map[string]any) + if !contains(required, "placement_offerings") || list["minItems"] != 1 { + t.Fatalf("missing nonempty placement requirement: %v", then) + } + } +} diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index 2b6ebcfc..b9464acf 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -31,10 +31,15 @@ func snapshotServer(t *testing.T) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { + case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap/actions/stop"): + io.WriteString(w, `{"id":"exp_snap","display_name":"Price paywall","status":"stopped"}`) + case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap/actions/start"): + io.WriteString(w, `{"object":"experiment","id":"exp_snap","display_name":"New paywall","status":"running","created_at":1784297950368,"updated_at":1784297950368}`) + case strings.HasSuffix(r.URL.Path, "/experiments"): io.WriteString(w, `{"items":[{"id":"exp_snap","display_name":"Four variants","status":"draft","offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"},"offering_c":{"id":"ofrng_c"},"offering_d":{"id":"ofrng_d"}}],"next_page":null}`) case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap"): - io.WriteString(w, `{"object":"experiment","id":"exp_snap","display_name":"Price — paywall","status":"paused","created_at":1784297950368,"updated_at":1784297950368,"total_running_time_seconds":1989824,"enrollment_percentage":50,"offering_a":{"id":"ofrng_control","display_name":"Control"},"offering_b":{"id":"ofrng_treatment","display_name":"Treatment"},"targeting_conditions":[{"field":"platform","operator":"in","value":["ios"]}],"placements":{"placement_offerings":[{"placement_identifier":"onboarding","offering_a":{"id":"ofrng_control"},"offering_b":{"id":"ofrng_treatment"}}]},"primary_metric":"initial_conversion_rate","secondary_metrics":["realized_ltv_per_customer","trial_conversion_rate"],"experiment_duration_settings":{"chance_to_win_percentage":95}}`) + io.WriteString(w, `{"object":"experiment","id":"exp_snap","display_name":"Price — paywall","status":"draft","created_at":1784297950368,"updated_at":1784297950368,"total_running_time_seconds":1989824,"enrollment_percentage":50,"offering_a":{"id":"ofrng_control","display_name":"Control","paywall_id":"pw_control"},"offering_b":{"id":"ofrng_treatment","display_name":"Treatment","paywall_id":"pw_treatment"},"offering_c":{"id":"ofrng_extra"},"targeting_conditions":[{"field":"platform","operator":"in","value":["ios"]}],"placements":{"placement_offerings":[{"placement_identifier":"onboarding","offering_a":{"id":"ofrng_control"},"offering_b":{"id":"ofrng_treatment"}}]},"experiment_type":"paywall_design","primary_metric":"initial_conversion_rate","secondary_metrics":["realized_ltv_per_customer","trial_conversion_rate"],"enrollment_mode":"only_new","experiment_duration_settings":{"chance_to_win_percentage":95},"conflicts":[{"id":"exp_other"}]}`) case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap/results"): io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[{"object":"experiment_results_section","section":"Conversion","metrics":[{"name":"trial_conversion_rate","unit":"%"}],"segments":[{"object":"experiment_results_segment","id":"total","display_name":"Total","is_total":true}],"values":[{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Control","value":12.5,"change":null,"credible_interval":null,"lift_credible_interval":null},{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Treatment","value":15.0,"change":20,"credible_interval":null,"lift_credible_interval":null}]}],"statistics":[{"object":"experiment_results_statistic","metric_name":"trial_conversion_rate","variants":[{"object":"experiment_results_variant_statistic","name":"Control","chance_to_win":null,"chance_to_win_status":"unavailable","credible_interval_lower":0.1,"credible_interval_upper":0.15,"credible_interval_status":"ok","lift_credible_interval_lower":null,"lift_credible_interval_upper":null,"lift_credible_interval_status":"unavailable"},{"object":"experiment_results_variant_statistic","name":"Treatment","chance_to_win":0.98,"chance_to_win_status":"ok","credible_interval_lower":0.11,"credible_interval_upper":0.18,"credible_interval_status":"ok","lift_credible_interval_lower":-5,"lift_credible_interval_upper":22,"lift_credible_interval_status":"ok"}]}],"predicted_ltv":null}`) case strings.HasSuffix(r.URL.Path, "/offerings/ofrng_snap"): @@ -70,6 +75,8 @@ func TestOutputSnapshots(t *testing.T) { {"experiments-list", []string{"experiments", "list", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"experiments-show", []string{"experiments", "show", "exp_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"experiments-results", []string{"experiments", "results", "exp_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, + {"experiments-stop", []string{"experiments", "stop", "exp_snap", "--yes", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, + {"experiments-start", []string{"experiments", "start", "exp_snap", "--yes", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"apps-list", []string{"apps", "list", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"apps-list-all-projects", []string{"apps", "list", "--all-projects", "--bundle-id", "com.example.moodly", "--no-input", "--api-key", "sk_snap"}}, {"error-not-found", []string{"offerings", "show", "ofrng_missing", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, diff --git a/internal/cli/testdata/snapshots/experiments-show.golden b/internal/cli/testdata/snapshots/experiments-show.golden index e7f18c55..4b0ac4bf 100644 --- a/internal/cli/testdata/snapshots/experiments-show.golden +++ b/internal/cli/testdata/snapshots/experiments-show.golden @@ -1,10 +1,11 @@ $ rc experiments show exp_snap --no-input --project-id proj_snap --api-key sk_snap ▍ Price — paywall - exp_snap · paused + exp_snap · draft Variants - Control: ofrng_control (Control) - Treatment: ofrng_treatment (Treatment) + Control: ofrng_control (Control) · paywall pw_control + Treatment: ofrng_treatment (Treatment) · paywall pw_treatment + Treatment C: ofrng_extra Audience Condition 1: platform in ios @@ -14,10 +15,15 @@ Placements Setup Enrollment: 50% + Enrollment mode: only_new + Type: paywall_design Primary metric: initial_conversion_rate Secondary metrics: realized_ltv_per_customer, trial_conversion_rate Duration settings: chance to win 95% Timeline Running time: 23d 0h 43m + +Conflicts + Experiments: exp_other Use --json for the complete API response. diff --git a/internal/cli/testdata/snapshots/experiments-start.golden b/internal/cli/testdata/snapshots/experiments-start.golden new file mode 100644 index 00000000..817f1077 --- /dev/null +++ b/internal/cli/testdata/snapshots/experiments-start.golden @@ -0,0 +1,30 @@ +$ rc experiments start exp_snap --yes --no-input --project-id proj_snap --api-key sk_snap + +▍ Experiment — Price — paywall + Start enrolling eligible customers in the experiment. + + Status draft + Control ofrng_control (Control) · paywall pw_control + Treatment ofrng_treatment (Treatment) · paywall pw_treatment + Treatment C ofrng_extra + Enrollment 50% + Condition 1 platform in ios + Placement onboarding Control ofrng_control · Treatment ofrng_treatment + Type paywall_design + Primary metric initial_conversion_rate + Secondary metrics realized_ltv_per_customer, trial_conversion_rate + Enrollment mode only_new + Duration chance to win 95% + +▐ Experiment conflicts: exp_other + + +▍ Plan +· 1. Start the experiment and enroll eligible customers +· Starting experiment… +✓ Started experiment exp_snap +▍ New paywall + + ID: exp_snap + Status: running + rc experiments results exp_snap diff --git a/internal/cli/testdata/snapshots/experiments-stop.golden b/internal/cli/testdata/snapshots/experiments-stop.golden new file mode 100644 index 00000000..8f11dd68 --- /dev/null +++ b/internal/cli/testdata/snapshots/experiments-stop.golden @@ -0,0 +1,42 @@ +$ rc experiments stop exp_snap --yes --no-input --project-id proj_snap --api-key sk_snap + +▍ Experiment — Price — paywall + Permanently stop the experiment. + + Status draft + Control ofrng_control (Control) · paywall pw_control + Treatment ofrng_treatment (Treatment) · paywall pw_treatment + Treatment C ofrng_extra + Enrollment 50% + Condition 1 platform in ios + Placement onboarding Control ofrng_control · Treatment ofrng_treatment + Type paywall_design + Primary metric initial_conversion_rate + Secondary metrics realized_ltv_per_customer, trial_conversion_rate + Enrollment mode only_new + Duration chance to win 95% + +▐ Experiment conflicts: exp_other + + +▍ Plan +· 1. Stop the experiment permanently +✓ Stopped experiment exp_snap +▍ Price paywall + exp_snap · stopped + +Variants + none + +Audience + Audience: All eligible customers + +Placements + No placement overrides + +Setup + none + +Timeline + Created: + Use --json for the complete API response. diff --git a/internal/output/output.go b/internal/output/output.go index 0f884f56..18fc1790 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -524,7 +524,7 @@ func (r *Renderer) Notice(lines ...string) { fmt.Fprintln(r.stderr) bar := lipgloss.NewStyle().Foreground(InfoBlue).Bold(true) for _, line := range lines { - fmt.Fprintln(r.stderr, r.style(bar, "▐ ")+line) + fmt.Fprintln(r.stderr, r.style(bar, "▐ ")+SanitizeLine(line)) } fmt.Fprintln(r.stderr) } diff --git a/internal/output/sanitize_test.go b/internal/output/sanitize_test.go index 02c7c147..21e947a3 100644 --- a/internal/output/sanitize_test.go +++ b/internal/output/sanitize_test.go @@ -129,6 +129,16 @@ func TestField_SanitizesValueAndNoteSeparately(t *testing.T) { } } +func TestNoticeSanitizesLine(t *testing.T) { + var out, errBuf bytes.Buffer + r := NewRenderer(&out, &errBuf, false, true, false, "") + r.Notice("Conflict \x1b]0;title\x07name\nnext") + s := errBuf.String() + if strings.ContainsAny(s, "\x1b\x07") || !strings.Contains(s, "Conflict ]0;titlename next") { + t.Errorf("notice output was not sanitized: %q", s) + } +} + func TestRenderFormat_EscapesControlBytes(t *testing.T) { var out, errBuf bytes.Buffer r := NewRenderer(&out, &errBuf, true, true, false, ".data.items[]")