From 339f4a448b0cb2d917ffcd4b4c3db5695c91e176 Mon Sep 17 00:00:00 2001 From: Josh Holtz Date: Fri, 25 Sep 2026 09:16:59 -0500 Subject: [PATCH 1/7] feat(experiments): add read commands and development API routes --- docs/command-surface.md | 18 ++ docs/previews/experiments-results.svg | 24 ++ docs/specs/cli-coverage.yaml | 7 + docs/specs/v2-beta-overlay.yaml | 265 ++++++++++++++++++ internal/api/client.go | 4 +- internal/api/experiments.go | 159 +++++++++++ internal/api/experiments_test.go | 57 ++++ internal/api/paths_gen.go | 12 + internal/cli/experiments.go | 185 ++++++++++++ internal/cli/experiments_test.go | 52 ++++ internal/cli/root.go | 1 + internal/cli/snapshot_test.go | 3 + .../snapshots/experiments-results.golden | 9 + scripts/seed-beta-overlay.py | 2 + 14 files changed, 797 insertions(+), 1 deletion(-) create mode 100644 docs/previews/experiments-results.svg create mode 100644 internal/api/experiments.go create mode 100644 internal/api/experiments_test.go create mode 100644 internal/cli/experiments.go create mode 100644 internal/cli/experiments_test.go create mode 100644 internal/cli/testdata/snapshots/experiments-results.golden diff --git a/docs/command-surface.md b/docs/command-surface.md index 235345a7..3b10a4dd 100644 --- a/docs/command-surface.md +++ b/docs/command-surface.md @@ -198,6 +198,24 @@ rc paywalls delete [--force] # attached/published pa # Flows (paywalls, web funnels, and multi-step experiences) rc flows list [--type paywall|web-funnel|general] [--exclude-type ...] # experimental (hidden from --help until Checkpoint rules roll out); repeatable; shows type and publish state +# Experiments — development v2 endpoints, normal discoverable CLI surface +rc experiments list [--status ] +rc experiments show +rc experiments results [--platform

] [--country ] [--exposure-status ] [--currency ] +rc experiments create --name --control --treatment --enrollment [--config ] +rc experiments update --config # partial update; file or - for stdin +rc experiments delete # draft only; confirmation/--yes +rc experiments start # approval gate: confirmation/--yes; shows current configuration first +rc experiments pause +rc experiments resume +rc experiments stop # permanent; confirmation/--yes + +# Targeting rules — development v2 endpoints +rc targeting list [--state ] +rc targeting show +rc targeting create --config # structured rule, initially inactive unless configured otherwise +rc targeting update --config # partial update +rc targeting delete # confirmation/--yes # Rico (AI assistant) rc rico [message] # streaming chat window in a TTY (--plain for a line loop) diff --git a/docs/previews/experiments-results.svg b/docs/previews/experiments-results.svg new file mode 100644 index 00000000..be1c75a1 --- /dev/null +++ b/docs/previews/experiments-results.svg @@ -0,0 +1,24 @@ + + + + + +$ rc experiments results exp_snap --no-input --project-id proj_snap --api-key sk_snap + +▍ Experiment results +  Currency                    USD + +▍ Conversion +METRIC                 VARIANT    VALUE  UNIT  CHANGE  +trial_conversion_rate  Control    12.5   %     —     +trial_conversion_rate  Treatment  15     %     +20.00% + + + diff --git a/docs/specs/cli-coverage.yaml b/docs/specs/cli-coverage.yaml index a925bde0..2dad4f78 100644 --- a/docs/specs/cli-coverage.yaml +++ b/docs/specs/cli-coverage.yaml @@ -235,6 +235,13 @@ endpoints: path: /projects/{project_id}/audiences - method: GET path: /projects/{project_id}/audiences/{audience_id} + # Experiments (development-status; provided by the beta overlay) + - method: GET + path: /projects/{project_id}/experiments + - method: GET + path: /projects/{project_id}/experiments/{experiment_id} + - method: GET + path: /projects/{project_id}/experiments/{experiment_id}/results # Audit - method: GET diff --git a/docs/specs/v2-beta-overlay.yaml b/docs/specs/v2-beta-overlay.yaml index c6589430..71e509e1 100644 --- a/docs/specs/v2-beta-overlay.yaml +++ b/docs/specs/v2-beta-overlay.yaml @@ -990,6 +990,271 @@ paths: responses: '200': description: OK + /projects/{project_id}/experiments: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + get: + summary: Get a list of experiments + operationId: list-experiments + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read + responses: + '200': + description: OK + post: + summary: Create an experiment + operationId: create-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/priority_list: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + put: + summary: Update the experiment priority list + operationId: update-experiment-priority-list + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + delete: + summary: Delete an experiment + operationId: delete-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + get: + summary: Get an experiment + operationId: get-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read + responses: + '200': + description: OK + post: + summary: Update an experiment + operationId: update-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}/actions/pause: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + post: + summary: Pause an experiment + operationId: pause-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}/actions/resume: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + post: + summary: Resume an experiment + operationId: resume-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}/actions/start: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + post: + summary: Start an experiment + operationId: start-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}/actions/stop: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + post: + summary: Stop an experiment + operationId: stop-experiment + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:experiments:read_write + responses: + '200': + description: OK + /projects/{project_id}/experiments/{experiment_id}/results: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: experiment_id + in: path + required: true + schema: + type: string + get: + summary: Get experiment results + operationId: get-experiment-results + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - charts_metrics:experiment_results:read + responses: + '200': + description: OK + /projects/{project_id}/targeting_rules: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + get: + summary: Get a list of targeting rules + operationId: list-targeting-rules + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:targeting_rules:read + responses: + '200': + description: OK + post: + summary: Create a targeting rule + operationId: create-targeting-rule + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:targeting_rules:read_write + responses: + '200': + description: OK + /projects/{project_id}/targeting_rules/{targeting_rule_id}: + parameters: + - name: project_id + in: path + required: true + schema: + type: string + - name: targeting_rule_id + in: path + required: true + schema: + type: string + delete: + summary: Delete a targeting rule + operationId: delete-targeting-rule + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:targeting_rules:read_write + responses: + '200': + description: OK + get: + summary: Get a targeting rule + operationId: get-targeting-rule + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:targeting_rules:read + responses: + '200': + description: OK + post: + summary: Update a targeting rule + operationId: update-targeting-rule + x-release-status: development + x-revenuecat-release-gatekeeping: false + x-scopes: + - project_configuration:targeting_rules:read_write + responses: + '200': + description: OK components: schemas: Currency: diff --git a/internal/api/client.go b/internal/api/client.go index ab4d3967..1ba3088e 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -57,6 +57,7 @@ type Client struct { Entitlements *EntitlementsService Flows *FlowsService Offerings *OfferingsService + Experiments *ExperimentsService Packages *PackagesService Products *ProductsService Subscriptions *SubscriptionsService @@ -72,7 +73,7 @@ type Client struct { Apps *AppsService StoreStatePlans *StoreStatePlansService StoreState *StoreStateService - // add as we go: Discounts, Experiments, VirtualCurrencies catalog + // add as we go: Discounts, VirtualCurrencies catalog } func NewClient(opts Options) *Client { @@ -103,6 +104,7 @@ func NewClient(opts Options) *Client { c.Entitlements = &EntitlementsService{c: c} c.Flows = &FlowsService{c: c} c.Offerings = &OfferingsService{c: c} + c.Experiments = &ExperimentsService{c: c} c.Packages = &PackagesService{c: c} c.Products = &ProductsService{c: c} c.Subscriptions = &SubscriptionsService{c: c} diff --git a/internal/api/experiments.go b/internal/api/experiments.go new file mode 100644 index 00000000..72a0d4b5 --- /dev/null +++ b/internal/api/experiments.go @@ -0,0 +1,159 @@ +package api + +import ( + "context" + "net/http" + "net/url" + "strconv" +) + +type ExperimentsService struct{ c *Client } + +type ExperimentOffering struct { + ID string `json:"id"` + Identifier string `json:"identifier,omitempty"` + DisplayName string `json:"display_name,omitempty"` + PaywallID string `json:"paywall_id,omitempty"` +} + +type Experiment struct { + Object string `json:"object"` + ID string `json:"id"` + DisplayName string `json:"display_name"` + Status string `json:"status"` + CreatedAt Millis `json:"created_at"` + UpdatedAt Millis `json:"updated_at"` + StartedAt *Millis `json:"started_at,omitempty"` + StoppedAt *Millis `json:"stopped_at,omitempty"` + PausedAt *Millis `json:"paused_at,omitempty"` + EnrollmentPercent *int `json:"enrollment_percentage,omitempty"` + Priority *int `json:"priority,omitempty"` + OfferingA *ExperimentOffering `json:"offering_a,omitempty"` + OfferingB *ExperimentOffering `json:"offering_b,omitempty"` + OfferingC *ExperimentOffering `json:"offering_c,omitempty"` + OfferingD *ExperimentOffering `json:"offering_d,omitempty"` + Notes *string `json:"notes,omitempty"` + ExperimentType *string `json:"experiment_type,omitempty"` + PrimaryMetric *string `json:"primary_metric,omitempty"` + SecondaryMetrics []string `json:"secondary_metrics,omitempty"` + EnrollmentMode *string `json:"enrollment_mode,omitempty"` + AudienceID *string `json:"audience_id,omitempty"` + TargetingConditions []any `json:"targeting_conditions,omitempty"` + Placements any `json:"placements,omitempty"` + Conflicts []any `json:"conflicts,omitempty"` +} + +type ListExperimentsOptions struct { + Status string + Limit int + StartingAfter string +} + +func (s *ExperimentsService) List(ctx context.Context, projectID string, opts ListExperimentsOptions) (*Page[Experiment], error) { + q := url.Values{} + if opts.Status != "" { + q.Set("status", opts.Status) + } + if opts.Limit > 0 { + q.Set("limit", strconv.Itoa(opts.Limit)) + } + if opts.StartingAfter != "" { + q.Set("starting_after", opts.StartingAfter) + } + path := pathExperiments(projectID) + if len(q) > 0 { + path += "?" + q.Encode() + } + var out Page[Experiment] + err := s.c.do(ctx, http.MethodGet, path, nil, &out) + return &out, err +} + +func (s *ExperimentsService) Get(ctx context.Context, projectID, id string) (*Experiment, error) { + var out Experiment + err := s.c.do(ctx, http.MethodGet, pathExperiment(projectID, id), nil, &out) + return &out, err +} + +type ExperimentResults struct { + Object string `json:"object"` + Sections []ExperimentResultsSection `json:"sections"` + Currency string `json:"currency"` + Statistics []ExperimentResultStatistic `json:"statistics"` + PredictedLTV *ExperimentPredictedLTV `json:"predicted_ltv"` +} + +type ExperimentResultsSection struct { + Object string `json:"object"` + Section string `json:"section"` + Description *string `json:"description,omitempty"` + Metrics []ExperimentResultMetric `json:"metrics"` + Segments []ExperimentResultSegment `json:"segments"` + Values []ExperimentResultValue `json:"values"` +} + +type ExperimentResultMetric struct { + Name string `json:"name"` + Unit string `json:"unit,omitempty"` + Effect string `json:"effect,omitempty"` +} + +type ExperimentResultSegment struct { + Object string `json:"object"` + ID string `json:"id"` + DisplayName string `json:"display_name"` + IsTotal bool `json:"is_total"` +} + +type ExperimentResultValue struct { + Object string `json:"object"` + Metric int `json:"metric"` + Segment int `json:"segment"` + Variant string `json:"variant"` + Value *float64 `json:"value"` + Change *float64 `json:"change"` + CredibleInterval any `json:"credible_interval"` + LiftCredibleInterval any `json:"lift_credible_interval"` +} + +type ExperimentResultStatistic struct { + Object string `json:"object"` + MetricName string `json:"metric_name"` + Variants []any `json:"variants"` +} + +type ExperimentPredictedLTV struct { + Object string `json:"object"` + PredictedWinnerVariant string `json:"predicted_winner_variant"` + Confidence int `json:"confidence"` +} + +type ExperimentResultsOptions struct { + Platform string + Country string + ExposureStatus string + Currency string +} + +func (s *ExperimentsService) Results(ctx context.Context, projectID, id string, opts ExperimentResultsOptions) (*ExperimentResults, error) { + q := url.Values{} + if opts.Platform != "" { + q.Set("platform", opts.Platform) + } + if opts.Country != "" { + q.Set("country", opts.Country) + } + if opts.ExposureStatus != "" { + q.Set("exposure_status", opts.ExposureStatus) + } + if opts.Currency != "" { + q.Set("currency", opts.Currency) + } + path := pathExperimentResults(projectID, id) + if len(q) > 0 { + path += "?" + q.Encode() + } + var out ExperimentResults + err := s.c.do(ctx, http.MethodGet, path, nil, &out) + return &out, err +} diff --git a/internal/api/experiments_test.go b/internal/api/experiments_test.go new file mode 100644 index 00000000..430268ee --- /dev/null +++ b/internal/api/experiments_test.go @@ -0,0 +1,57 @@ +package api_test + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/revenuecat/cli/internal/api" +) + +func TestExperimentsReadRoutesAndFilters(t *testing.T) { + requests := []string{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests = append(requests, r.Method+" "+r.URL.RequestURI()) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/projects/proj/experiments": + fmt.Fprint(w, `{"object":"list","items":[{"object":"experiment","id":"exp1","display_name":"Onboarding","status":"running","created_at":1,"updated_at":2,"offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"}}],"next_page":null,"url":"/projects/proj/experiments"}`) + case "/projects/proj/experiments/exp1": + fmt.Fprint(w, `{"object":"experiment","id":"exp1","display_name":"Onboarding","status":"running","created_at":1,"updated_at":2}`) + case "/projects/proj/experiments/exp1/results": + fmt.Fprint(w, `{"object":"experiment_results","currency":"EUR","sections":[],"statistics":[],"predicted_ltv":null}`) + default: + t.Errorf("unexpected path %s", r.URL.Path) + } + })) + t.Cleanup(srv.Close) + client := api.NewClient(api.Options{APIKey: "sk_test", BaseURL: srv.URL}) + ctx := context.Background() + page, err := client.Experiments.List(ctx, "proj", api.ListExperimentsOptions{Status: "running", Limit: 5, StartingAfter: "exp0"}) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 || page.Items[0].OfferingB.ID != "ofrng_b" { + t.Fatalf("unexpected page: %+v", page) + } + if _, err := client.Experiments.Get(ctx, "proj", "exp1"); err != nil { + t.Fatal(err) + } + results, err := client.Experiments.Results(ctx, "proj", "exp1", api.ExperimentResultsOptions{Platform: "ios", Country: "US", ExposureStatus: "exposed", Currency: "EUR"}) + if err != nil { + t.Fatal(err) + } + if results.Currency != "EUR" { + t.Fatalf("currency = %q", results.Currency) + } + want := []string{ + "GET /projects/proj/experiments?limit=5&starting_after=exp0&status=running", + "GET /projects/proj/experiments/exp1", + "GET /projects/proj/experiments/exp1/results?country=US¤cy=EUR&exposure_status=exposed&platform=ios", + } + if fmt.Sprint(requests) != fmt.Sprint(want) { + t.Fatalf("requests = %v, want %v", requests, want) + } +} diff --git a/internal/api/paths_gen.go b/internal/api/paths_gen.go index c82b3342..6ea7ea2f 100644 --- a/internal/api/paths_gen.go +++ b/internal/api/paths_gen.go @@ -134,6 +134,18 @@ func pathEntitlements(projectID string) string { return encodePath("projects", projectID, "entitlements") } +func pathExperiment(projectID string, experimentID string) string { + return encodePath("projects", projectID, "experiments", experimentID) +} + +func pathExperimentResults(projectID string, experimentID string) string { + return encodePath("projects", projectID, "experiments", experimentID, "results") +} + +func pathExperiments(projectID string) string { + return encodePath("projects", projectID, "experiments") +} + func pathFlows(projectID string) string { return encodePath("projects", projectID, "flows") } diff --git a/internal/cli/experiments.go b/internal/cli/experiments.go new file mode 100644 index 00000000..20f4a741 --- /dev/null +++ b/internal/cli/experiments.go @@ -0,0 +1,185 @@ +package cli + +import ( + "fmt" + "strconv" + + "github.com/spf13/cobra" + + "github.com/revenuecat/cli/internal/api" + "github.com/revenuecat/cli/internal/output" +) + +func newExperimentsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "experiments", + Short: "Run and analyze offering experiments", + Long: "Inspect experiments, compare their results, and manage their lifecycle.", + } + cmd.AddCommand(newExperimentsListCmd(), newExperimentsShowCmd(), newExperimentsResultsCmd()) + return cmd +} + +func newExperimentsListCmd() *cobra.Command { + var status, startingAfter string + var limit int + cmd := &cobra.Command{ + Use: "list", + Short: "List experiments", + Example: " rc experiments list --status running\n rc experiments list --json", + RunE: func(cmd *cobra.Command, _ []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + page, err := client.Experiments.List(cmd.Context(), projectID, api.ListExperimentsOptions{ + Status: status, Limit: limit, StartingAfter: startingAfter, + }) + if err != nil { + return err + } + rows := make([][]string, 0, len(page.Items)) + for _, experiment := range page.Items { + control, treatment := "", "" + if experiment.OfferingA != nil { + control = experiment.OfferingA.ID + } + if experiment.OfferingB != nil { + treatment = experiment.OfferingB.ID + } + rows = append(rows, []string{experiment.ID, experiment.DisplayName, experiment.Status, control, treatment}) + } + return rt.Out.RenderTable(output.Table{ + Columns: []string{"ID", "NAME", "STATUS", "CONTROL", "TREATMENT"}, + Rows: rows, Raw: page, + }) + }, + } + cmd.Flags().StringVar(&status, "status", "", "filter by draft, running, paused, or stopped") + cmd.Flags().IntVar(&limit, "limit", 20, "maximum experiments to return (1–100)") + cmd.Flags().StringVar(&startingAfter, "starting-after", "", "pagination cursor from the previous page") + return cmd +} + +func experimentPickerItems(cmd *cobra.Command, client *api.Client, projectID string) ([]PickerItem, error) { + page, err := client.Experiments.List(cmd.Context(), projectID, api.ListExperimentsOptions{}) + if err != nil { + return nil, err + } + items := make([]PickerItem, len(page.Items)) + for i, experiment := range page.Items { + items[i] = PickerItem{ID: experiment.ID, Label: fmt.Sprintf("%s (%s)", experiment.DisplayName, experiment.Status)} + } + return items, nil +} + +func newExperimentsShowCmd() *cobra.Command { + return &cobra.Command{ + Use: "show [id]", + Short: "Show an experiment", + Example: " rc experiments show exp123 --json", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + id, err := requireID(rt, argAt(args, 0), "experiment", func() ([]PickerItem, error) { + return experimentPickerItems(cmd, client, projectID) + }) + if err != nil { + return err + } + experiment, err := client.Experiments.Get(cmd.Context(), projectID, id) + if err != nil { + return err + } + return rt.Out.Render(experiment) + }, + } +} + +func newExperimentsResultsCmd() *cobra.Command { + var opts api.ExperimentResultsOptions + cmd := &cobra.Command{ + Use: "results [id]", + Short: "Compare experiment variants and metrics", + Long: "Shows total-segment metrics in human output. Use --json for all sections, product segments, and statistics.", + Example: " rc experiments results exp123\n rc experiments results exp123 --json", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + rt := RuntimeFrom(cmd.Context()) + projectID, err := requireProject(rt) + if err != nil { + return err + } + client, err := rt.API() + if err != nil { + return err + } + id, err := requireID(rt, argAt(args, 0), "experiment", func() ([]PickerItem, error) { + return experimentPickerItems(cmd, client, projectID) + }) + if err != nil { + return err + } + results, err := client.Experiments.Results(cmd.Context(), projectID, id, opts) + if err != nil { + return err + } + if rt.Globals.JSON { + return rt.Out.Render(results) + } + return renderExperimentResults(rt, results) + }, + } + cmd.Flags().StringVar(&opts.Platform, "platform", "", "filter by platform") + cmd.Flags().StringVar(&opts.Country, "country", "", "filter by country code") + cmd.Flags().StringVar(&opts.ExposureStatus, "exposure-status", "", "filter by exposure status") + cmd.Flags().StringVar(&opts.Currency, "currency", "", "display currency for monetary metrics") + return cmd +} + +func renderExperimentResults(rt *Runtime, results *api.ExperimentResults) error { + rt.Out.Title("Experiment results") + rt.Out.Field("Currency", results.Currency) + for _, section := range results.Sections { + rows := make([][]string, 0) + for _, value := range section.Values { + if value.Metric < 0 || value.Metric >= len(section.Metrics) || value.Segment < 0 || value.Segment >= len(section.Segments) || !section.Segments[value.Segment].IsTotal { + continue + } + metric := section.Metrics[value.Metric] + shownValue, change := "—", "—" + if value.Value != nil { + shownValue = strconv.FormatFloat(*value.Value, 'f', -1, 64) + } + if value.Change != nil { + change = fmt.Sprintf("%+.2f%%", *value.Change) + } + rows = append(rows, []string{metric.Name, value.Variant, shownValue, metric.Unit, change}) + } + if len(rows) == 0 { + continue + } + rt.Out.Title(section.Section) + if err := rt.Out.RenderTable(output.Table{Columns: []string{"METRIC", "VARIANT", "VALUE", "UNIT", "CHANGE"}, Rows: rows}); err != nil { + return err + } + } + if results.PredictedLTV != nil { + rt.Out.Field("Predicted winner", results.PredictedLTV.PredictedWinnerVariant) + rt.Out.Field("Confidence", fmt.Sprintf("%d%%", results.PredictedLTV.Confidence)) + } + return nil +} diff --git a/internal/cli/experiments_test.go b/internal/cli/experiments_test.go new file mode 100644 index 00000000..bafbfd0b --- /dev/null +++ b/internal/cli/experiments_test.go @@ -0,0 +1,52 @@ +package cli_test + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestExperimentResultsJSONIncludesAllSegments(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/projects/proj/experiments/exp1/results" { + t.Errorf("unexpected route: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[{"object":"experiment_results_section","section":"Revenue","metrics":[{"name":"realized_ltv_per_customer"}],"segments":[{"object":"experiment_results_segment","id":"product","display_name":"Annual","is_total":false}],"values":[{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Treatment","value":12.5,"change":5,"credible_interval":null,"lift_credible_interval":null}]}],"statistics":[],"predicted_ltv":null}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + out, stderr, err := runAgentCmd(t, "experiments", "results", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--json", "--no-input") + if err != nil || stderr != "" { + t.Fatalf("err=%v stderr=%q", err, stderr) + } + var envelope struct { + Data struct { + Sections []struct { + Segments []struct { + ID string `json:"id"` + } `json:"segments"` + } `json:"sections"` + } `json:"data"` + SchemaVersion int `json:"schema_version"` + } + if err := json.Unmarshal([]byte(out), &envelope); err != nil { + t.Fatal(err) + } + if envelope.SchemaVersion != 1 || envelope.Data.Sections[0].Segments[0].ID != "product" { + t.Fatalf("unexpected envelope: %s", out) + } +} + +func TestExperimentsDiscoverable(t *testing.T) { + out, _, err := runCmd(t, "schema", "experiments", "results", "--json") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, `"path": "rc experiments results"`) || strings.Contains(out, `"experimental": true`) { + t.Fatalf("results command not discoverable as a default command: %s", out) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index d879faee..4b0e8777 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -156,6 +156,7 @@ Agent-friendly entrypoints: {newEntitlementsCmd(), "catalog"}, {newOfferingsCmd(), "catalog"}, {newCheckpointsCmd(), "catalog"}, + {newExperimentsCmd(), "revenue"}, {newProductsCmd(), "catalog"}, {newSubscriptionsCmd(), "revenue"}, {newPurchasesCmd(), "revenue"}, diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index 0586eb09..22f9b361 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -28,6 +28,8 @@ func snapshotServer(t *testing.T) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { + case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap/results"): + io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[{"object":"experiment_results_section","section":"Conversion","metrics":[{"name":"trial_conversion_rate","unit":"%"}],"segments":[{"object":"experiment_results_segment","id":"total","display_name":"Total","is_total":true}],"values":[{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Control","value":12.5,"change":null,"credible_interval":null,"lift_credible_interval":null},{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Treatment","value":15.0,"change":20,"credible_interval":null,"lift_credible_interval":null}]}],"statistics":[],"predicted_ltv":null}`) case strings.HasSuffix(r.URL.Path, "/offerings/ofrng_snap"): io.WriteString(w, `{"object":"offering","id":"ofrng_snap","lookup_key":"default","display_name":"Default","is_current":true,"created_at":1784297950368,"project_id":"proj_snap"}`) case strings.HasSuffix(r.URL.Path, "/apps/app_snap1"): @@ -58,6 +60,7 @@ func TestOutputSnapshots(t *testing.T) { {"version", []string{"version"}}, {"auth-status-logged-out", []string{"auth", "status", "--no-input"}}, {"offerings-show", []string{"offerings", "show", "ofrng_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, + {"experiments-results", []string{"experiments", "results", "exp_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"apps-list", []string{"apps", "list", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"apps-list-all-projects", []string{"apps", "list", "--all-projects", "--bundle-id", "com.example.moodly", "--no-input", "--api-key", "sk_snap"}}, {"error-not-found", []string{"offerings", "show", "ofrng_missing", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, diff --git a/internal/cli/testdata/snapshots/experiments-results.golden b/internal/cli/testdata/snapshots/experiments-results.golden new file mode 100644 index 00000000..a1a761b0 --- /dev/null +++ b/internal/cli/testdata/snapshots/experiments-results.golden @@ -0,0 +1,9 @@ +$ rc experiments results exp_snap --no-input --project-id proj_snap --api-key sk_snap + +▍ Experiment results + Currency USD + +▍ Conversion +METRIC VARIANT VALUE UNIT CHANGE +trial_conversion_rate Control 12.5 % — +trial_conversion_rate Treatment 15 % +20.00% diff --git a/scripts/seed-beta-overlay.py b/scripts/seed-beta-overlay.py index 6ddb70fd..7496fd9b 100644 --- a/scripts/seed-beta-overlay.py +++ b/scripts/seed-beta-overlay.py @@ -33,6 +33,8 @@ "/store_state/plans", "/products/{product_id}/prices", "/products/{product_id}/test_store_prices", + "/experiments", + "/targeting_rules", ] REF_RE = re.compile(r"#/components/([A-Za-z0-9]+)/([A-Za-z0-9_.-]+)") From 77abac54e3e31fcdd39a746d734296c664ca47b6 Mon Sep 17 00:00:00 2001 From: Josh Holtz Date: Fri, 25 Sep 2026 09:46:41 -0500 Subject: [PATCH 2/7] fix(experiments): improve read UX and isolate routes --- docs/command-surface.md | 14 --- docs/specs/v2-beta-overlay.yaml | 204 ------------------------------- internal/cli/experiments.go | 34 ++++-- internal/cli/experiments_test.go | 18 +++ scripts/seed-beta-overlay.py | 13 +- 5 files changed, 50 insertions(+), 233 deletions(-) diff --git a/docs/command-surface.md b/docs/command-surface.md index 3b10a4dd..bf871d47 100644 --- a/docs/command-surface.md +++ b/docs/command-surface.md @@ -202,20 +202,6 @@ rc flows list [--type paywall|web-funnel|general] [--exclude-type ...] # exper rc experiments list [--status ] rc experiments show rc experiments results [--platform

] [--country ] [--exposure-status ] [--currency ] -rc experiments create --name --control --treatment --enrollment [--config ] -rc experiments update --config # partial update; file or - for stdin -rc experiments delete # draft only; confirmation/--yes -rc experiments start # approval gate: confirmation/--yes; shows current configuration first -rc experiments pause -rc experiments resume -rc experiments stop # permanent; confirmation/--yes - -# Targeting rules — development v2 endpoints -rc targeting list [--state ] -rc targeting show -rc targeting create --config # structured rule, initially inactive unless configured otherwise -rc targeting update --config # partial update -rc targeting delete # confirmation/--yes # Rico (AI assistant) rc rico [message] # streaming chat window in a TTY (--plain for a line loop) diff --git a/docs/specs/v2-beta-overlay.yaml b/docs/specs/v2-beta-overlay.yaml index 71e509e1..e92934eb 100644 --- a/docs/specs/v2-beta-overlay.yaml +++ b/docs/specs/v2-beta-overlay.yaml @@ -1007,33 +1007,6 @@ paths: responses: '200': description: OK - post: - summary: Create an experiment - operationId: create-experiment - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:experiments:read_write - responses: - '200': - description: OK - /projects/{project_id}/experiments/priority_list: - parameters: - - name: project_id - in: path - required: true - schema: - type: string - put: - summary: Update the experiment priority list - operationId: update-experiment-priority-list - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:experiments:read_write - responses: - '200': - description: OK /projects/{project_id}/experiments/{experiment_id}: parameters: - name: project_id @@ -1046,16 +1019,6 @@ paths: required: true schema: type: string - delete: - summary: Delete an experiment - operationId: delete-experiment - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:experiments:read_write - responses: - '200': - description: OK get: summary: Get an experiment operationId: get-experiment @@ -1066,104 +1029,6 @@ paths: responses: '200': description: OK - post: - summary: Update an experiment - operationId: update-experiment - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:experiments:read_write - responses: - '200': - description: OK - /projects/{project_id}/experiments/{experiment_id}/actions/pause: - parameters: - - name: project_id - in: path - required: true - schema: - type: string - - name: experiment_id - in: path - required: true - schema: - type: string - post: - summary: Pause an experiment - operationId: pause-experiment - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:experiments:read_write - responses: - '200': - description: OK - /projects/{project_id}/experiments/{experiment_id}/actions/resume: - parameters: - - name: project_id - in: path - required: true - schema: - type: string - - name: experiment_id - in: path - required: true - schema: - type: string - post: - summary: Resume an experiment - operationId: resume-experiment - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:experiments:read_write - responses: - '200': - description: OK - /projects/{project_id}/experiments/{experiment_id}/actions/start: - parameters: - - name: project_id - in: path - required: true - schema: - type: string - - name: experiment_id - in: path - required: true - schema: - type: string - post: - summary: Start an experiment - operationId: start-experiment - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:experiments:read_write - responses: - '200': - description: OK - /projects/{project_id}/experiments/{experiment_id}/actions/stop: - parameters: - - name: project_id - in: path - required: true - schema: - type: string - - name: experiment_id - in: path - required: true - schema: - type: string - post: - summary: Stop an experiment - operationId: stop-experiment - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:experiments:read_write - responses: - '200': - description: OK /projects/{project_id}/experiments/{experiment_id}/results: parameters: - name: project_id @@ -1186,75 +1051,6 @@ paths: responses: '200': description: OK - /projects/{project_id}/targeting_rules: - parameters: - - name: project_id - in: path - required: true - schema: - type: string - get: - summary: Get a list of targeting rules - operationId: list-targeting-rules - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:targeting_rules:read - responses: - '200': - description: OK - post: - summary: Create a targeting rule - operationId: create-targeting-rule - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:targeting_rules:read_write - responses: - '200': - description: OK - /projects/{project_id}/targeting_rules/{targeting_rule_id}: - parameters: - - name: project_id - in: path - required: true - schema: - type: string - - name: targeting_rule_id - in: path - required: true - schema: - type: string - delete: - summary: Delete a targeting rule - operationId: delete-targeting-rule - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:targeting_rules:read_write - responses: - '200': - description: OK - get: - summary: Get a targeting rule - operationId: get-targeting-rule - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:targeting_rules:read - responses: - '200': - description: OK - post: - summary: Update a targeting rule - operationId: update-targeting-rule - x-release-status: development - x-revenuecat-release-gatekeeping: false - x-scopes: - - project_configuration:targeting_rules:read_write - responses: - '200': - description: OK components: schemas: Currency: diff --git a/internal/cli/experiments.go b/internal/cli/experiments.go index 20f4a741..f9ee1f78 100644 --- a/internal/cli/experiments.go +++ b/internal/cli/experiments.go @@ -21,7 +21,7 @@ func newExperimentsCmd() *cobra.Command { } func newExperimentsListCmd() *cobra.Command { - var status, startingAfter string + var status, cursor string var limit int cmd := &cobra.Command{ Use: "list", @@ -38,7 +38,7 @@ func newExperimentsListCmd() *cobra.Command { return err } page, err := client.Experiments.List(cmd.Context(), projectID, api.ListExperimentsOptions{ - Status: status, Limit: limit, StartingAfter: startingAfter, + Status: status, Limit: limit, StartingAfter: cursor, }) if err != nil { return err @@ -54,15 +54,19 @@ func newExperimentsListCmd() *cobra.Command { } rows = append(rows, []string{experiment.ID, experiment.DisplayName, experiment.Status, control, treatment}) } - return rt.Out.RenderTable(output.Table{ + if err := rt.Out.RenderTable(output.Table{ Columns: []string{"ID", "NAME", "STATUS", "CONTROL", "TREATMENT"}, Rows: rows, Raw: page, - }) + }); err != nil { + return err + } + hintMoreResults(rt, page) + return nil }, } cmd.Flags().StringVar(&status, "status", "", "filter by draft, running, paused, or stopped") cmd.Flags().IntVar(&limit, "limit", 20, "maximum experiments to return (1–100)") - cmd.Flags().StringVar(&startingAfter, "starting-after", "", "pagination cursor from the previous page") + cmd.Flags().StringVar(&cursor, "cursor", "", "item ID to start after (pagination)") return cmd } @@ -104,7 +108,13 @@ func newExperimentsShowCmd() *cobra.Command { if err != nil { return err } - return rt.Out.Render(experiment) + if err := rt.Out.Render(experiment); err != nil { + return err + } + if !rt.Globals.JSON { + rt.Out.Hint("Use --json to inspect targeting, placements, and all variant details.") + } + return nil }, } } @@ -153,6 +163,7 @@ func newExperimentsResultsCmd() *cobra.Command { func renderExperimentResults(rt *Runtime, results *api.ExperimentResults) error { rt.Out.Title("Experiment results") rt.Out.Field("Currency", results.Currency) + shownSections := 0 for _, section := range results.Sections { rows := make([][]string, 0) for _, value := range section.Values { @@ -172,13 +183,22 @@ func renderExperimentResults(rt *Runtime, results *api.ExperimentResults) error if len(rows) == 0 { continue } + shownSections++ rt.Out.Title(section.Section) if err := rt.Out.RenderTable(output.Table{Columns: []string{"METRIC", "VARIANT", "VALUE", "UNIT", "CHANGE"}, Rows: rows}); err != nil { return err } } + if shownSections == 0 { + rt.Out.Info("No total-segment metrics available.") + rt.Out.Hint("Use --json to inspect all segments and statistics.") + } if results.PredictedLTV != nil { - rt.Out.Field("Predicted winner", results.PredictedLTV.PredictedWinnerVariant) + winner := map[string]string{"a": "Control", "b": "Treatment", "c": "Treatment C", "d": "Treatment D"}[results.PredictedLTV.PredictedWinnerVariant] + if winner == "" { + winner = results.PredictedLTV.PredictedWinnerVariant + } + rt.Out.Field("Predicted winner", winner) rt.Out.Field("Confidence", fmt.Sprintf("%d%%", results.PredictedLTV.Confidence)) } return nil diff --git a/internal/cli/experiments_test.go b/internal/cli/experiments_test.go index bafbfd0b..03472d2c 100644 --- a/internal/cli/experiments_test.go +++ b/internal/cli/experiments_test.go @@ -41,6 +41,24 @@ func TestExperimentResultsJSONIncludesAllSegments(t *testing.T) { } } +func TestExperimentResultsHumanExplainsMissingTotals(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[{"section":"Revenue","metrics":[{"name":"revenue"}],"segments":[{"id":"product","display_name":"Annual","is_total":false}],"values":[{"metric":0,"segment":0,"variant":"Treatment","value":12.5}]}],"statistics":[],"predicted_ltv":{"predicted_winner_variant":"b","confidence":85}}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + out, stderr, err := runAgentCmd(t, "experiments", "results", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"No total-segment metrics available", "Use --json", "Predicted winner", "Treatment"} { + if !strings.Contains(out+stderr, want) { + t.Fatalf("missing %q in output: stdout=%s stderr=%s", want, out, stderr) + } + } +} + func TestExperimentsDiscoverable(t *testing.T) { out, _, err := runCmd(t, "schema", "experiments", "results", "--json") if err != nil { diff --git a/scripts/seed-beta-overlay.py b/scripts/seed-beta-overlay.py index 7496fd9b..02c4ad23 100644 --- a/scripts/seed-beta-overlay.py +++ b/scripts/seed-beta-overlay.py @@ -2,8 +2,8 @@ """Seed docs/specs/v2-beta-overlay.yaml from a source OpenAPI spec. The public spec we fetch from docs is release-filtered and omits endpoints the -CLI uses but that the backend marks `x-release-status: development` (the -store_state family). This extracts just those path keys — with request/response +CLI uses but that the backend marks `x-release-status: development`. +This extracts just those path keys — with request/response bodies stripped — into a small overlay that preprocess-spec.py merges on top of the public spec at codegen/diff time. The overlay declares that the paths exist so gen-paths builds path helpers and spec-diff tracks coverage; the CLI @@ -11,7 +11,7 @@ This is a *seeding* tool, not part of CI: run it once against a local checkout of the dev spec, then hand-maintain the result. Re-run to reseed when the -upstream store_state schema changes. +upstream development schema changes. python3 scripts/seed-beta-overlay.py \ path/to/source-openapi.yaml \ @@ -26,15 +26,13 @@ import yaml # Only pull paths whose key contains one of these — the development-status -# endpoints the CLI depends on: per-product store_state, the store_state plans -# tree, and product price management. Keep this to paths the CLI actually calls. +# endpoints the CLI depends on. Keep this to paths the CLI actually calls. PATH_SUBSTRINGS = [ "/products/{product_id}/store_state", "/store_state/plans", "/products/{product_id}/prices", "/products/{product_id}/test_store_prices", "/experiments", - "/targeting_rules", ] REF_RE = re.compile(r"#/components/([A-Za-z0-9]+)/([A-Za-z0-9_.-]+)") @@ -73,8 +71,7 @@ def main(): print("no matching paths found — check PATH_SUBSTRINGS", file=sys.stderr) sys.exit(1) - # The CLI hand-writes the types for these development-status endpoints (see - # internal/api/store_state_*.go and products.go). The overlay only needs to + # The CLI hand-writes the types for these development-status endpoints. The overlay only needs to # declare that the paths exist — so gen-paths builds path helpers and # spec-diff tracks coverage — not to generate models. Strip request/response # bodies so oapi-codegen emits nothing for them. From 718fbe6d781c06a03fe4cbbd3bec08ac41a97c34 Mon Sep 17 00:00:00 2001 From: Josh Holtz Date: Fri, 25 Sep 2026 12:17:08 -0500 Subject: [PATCH 3/7] fix(experiments): normalize filters and clarify human output --- docs/command-surface.md | 4 +- docs/previews/apps-list-all-projects.svg | 8 +- docs/previews/apps-list.svg | 8 +- docs/previews/experiments-results.svg | 9 +- docs/previews/experiments-show.svg | 38 +++ internal/api/experiments.go | 83 ++++-- internal/cli/experiment_format.go | 249 ++++++++++++++++++ internal/cli/experiments.go | 126 +++++++-- internal/cli/experiments_test.go | 35 +++ internal/cli/snapshot_test.go | 3 + .../snapshots/apps-list-all-projects.golden | 4 +- .../cli/testdata/snapshots/apps-list.golden | 2 +- .../snapshots/experiments-results.golden | 5 +- .../snapshots/experiments-show.golden | 23 ++ internal/output/card.go | 8 +- internal/output/output.go | 9 +- internal/output/output_test.go | 29 ++ 17 files changed, 572 insertions(+), 71 deletions(-) create mode 100644 docs/previews/experiments-show.svg create mode 100644 internal/cli/experiment_format.go create mode 100644 internal/cli/testdata/snapshots/experiments-show.golden diff --git a/docs/command-surface.md b/docs/command-surface.md index bf871d47..18089165 100644 --- a/docs/command-surface.md +++ b/docs/command-surface.md @@ -200,8 +200,8 @@ rc paywalls delete [--force] # attached/published pa rc flows list [--type paywall|web-funnel|general] [--exclude-type ...] # experimental (hidden from --help until Checkpoint rules roll out); repeatable; shows type and publish state # Experiments — development v2 endpoints, normal discoverable CLI surface rc experiments list [--status ] -rc experiments show -rc experiments results [--platform

] [--country ] [--exposure-status ] [--currency ] +rc experiments show # human detail view shows variant offerings, targeting, placements, and readable run time +rc experiments results [--platform

] [--country ] [--exposure-status exposed|not_exposed] [--currency ] # human totals compare variants by metric # Rico (AI assistant) rc rico [message] # streaming chat window in a TTY (--plain for a line loop) diff --git a/docs/previews/apps-list-all-projects.svg b/docs/previews/apps-list-all-projects.svg index 9507c206..50c41aa5 100644 --- a/docs/previews/apps-list-all-projects.svg +++ b/docs/previews/apps-list-all-projects.svg @@ -1,6 +1,6 @@ - + + $ rc apps list --all-projects --bundle-id com.example.moodly --no-input --api-key sk_snap ! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set.   Fix it:  rc setup apple app_snap1 --project-id proj_snap  (interactive Apple sign-in with 2FA) -PROJECT     ID         NAME                        TYPE       CREATED     CREDENTIALS                     +PROJECT     ID         NAME                        TYPE       CREATED     CREDENTIALS                   proj_snap   app_snap1  Moodly (App Store)          app_store  2026-07-16  partial — run: rc setup apple -proj_snap2  app_snap3  Moodly Staging (App Store)  app_store  2026-07-16  ready                           +proj_snap2  app_snap3  Moodly Staging (App Store)  app_store  2026-07-16  ready                         diff --git a/docs/previews/apps-list.svg b/docs/previews/apps-list.svg index 46f07c72..5c092486 100644 --- a/docs/previews/apps-list.svg +++ b/docs/previews/apps-list.svg @@ -12,10 +12,10 @@ $ rc apps list --no-input --project-id proj_snap --api-key sk_snap ! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. -  Fix it:  rc apps apple setup app_snap1  (interactive Apple sign-in with 2FA) -ID         NAME                TYPE        CREATED     CREDENTIALS                          -app_snap1  Moodly (App Store)  app_store   2026-07-16  partial — run: rc apps apple setup -app_snap2  Test Store          test_store  2026-07-16  —                                  +  Fix it:  rc setup apple app_snap1  (interactive Apple sign-in with 2FA) +ID         NAME                TYPE        CREATED     CREDENTIALS                   +app_snap1  Moodly (App Store)  app_store   2026-07-16  partial — run: rc setup apple +app_snap2  Test Store          test_store  2026-07-16  —                             diff --git a/docs/previews/experiments-results.svg b/docs/previews/experiments-results.svg index be1c75a1..ecf310d0 100644 --- a/docs/previews/experiments-results.svg +++ b/docs/previews/experiments-results.svg @@ -1,6 +1,6 @@ - + + $ rc experiments results exp_snap --no-input --project-id proj_snap --api-key sk_snap @@ -16,9 +16,8 @@   Currency                    USD ▍ Conversion -METRIC                 VARIANT    VALUE  UNIT  CHANGE  -trial_conversion_rate  Control    12.5   %     —     -trial_conversion_rate  Treatment  15     %     +20.00% +METRIC                 CONTROL  TREATMENT  CHANGE   UNIT +trial_conversion_rate  12.5     15         +20.00%  %    diff --git a/docs/previews/experiments-show.svg b/docs/previews/experiments-show.svg new file mode 100644 index 00000000..882b24e0 --- /dev/null +++ b/docs/previews/experiments-show.svg @@ -0,0 +1,38 @@ + + + + + +$ rc experiments show exp_snap --no-input --project-id proj_snap --api-key sk_snap +  Use --json for the complete API response. +▍ Price — paywall +  exp_snap · paused + +Variants +  Control:    ofrng_control (Control) +  Treatment:  ofrng_treatment (Treatment) + +Audience +  Condition 1:  platform in ios + +Placements +  onboarding:  Control ofrng_control · Treatment ofrng_treatment + +Setup +  Enrollment:         50% +  Primary metric:     initial_conversion_rate +  Secondary metrics:  realized_ltv_per_customer, trial_conversion_rate +  Duration settings:  chance to win 95% + +Timeline +  Running time:  23d 0h 43m + + + diff --git a/internal/api/experiments.go b/internal/api/experiments.go index 72a0d4b5..23a3a52a 100644 --- a/internal/api/experiments.go +++ b/internal/api/experiments.go @@ -16,31 +16,72 @@ type ExperimentOffering struct { PaywallID string `json:"paywall_id,omitempty"` } -type Experiment struct { - Object string `json:"object"` - ID string `json:"id"` - DisplayName string `json:"display_name"` - Status string `json:"status"` - CreatedAt Millis `json:"created_at"` - UpdatedAt Millis `json:"updated_at"` - StartedAt *Millis `json:"started_at,omitempty"` - StoppedAt *Millis `json:"stopped_at,omitempty"` - PausedAt *Millis `json:"paused_at,omitempty"` - EnrollmentPercent *int `json:"enrollment_percentage,omitempty"` - Priority *int `json:"priority,omitempty"` +type ExperimentCondition struct { + Field string `json:"field"` + Operator string `json:"operator"` + Value any `json:"value"` + Context any `json:"context,omitempty"` +} + +type ExperimentPlacementOffering struct { + PlacementIdentifier string `json:"placement_identifier"` OfferingA *ExperimentOffering `json:"offering_a,omitempty"` OfferingB *ExperimentOffering `json:"offering_b,omitempty"` OfferingC *ExperimentOffering `json:"offering_c,omitempty"` OfferingD *ExperimentOffering `json:"offering_d,omitempty"` - Notes *string `json:"notes,omitempty"` - ExperimentType *string `json:"experiment_type,omitempty"` - PrimaryMetric *string `json:"primary_metric,omitempty"` - SecondaryMetrics []string `json:"secondary_metrics,omitempty"` - EnrollmentMode *string `json:"enrollment_mode,omitempty"` - AudienceID *string `json:"audience_id,omitempty"` - TargetingConditions []any `json:"targeting_conditions,omitempty"` - Placements any `json:"placements,omitempty"` - Conflicts []any `json:"conflicts,omitempty"` +} + +type ExperimentPlacements struct { + FallbackOfferingA *ExperimentOffering `json:"fallback_offering_a,omitempty"` + FallbackOfferingB *ExperimentOffering `json:"fallback_offering_b,omitempty"` + FallbackOfferingC *ExperimentOffering `json:"fallback_offering_c,omitempty"` + FallbackOfferingD *ExperimentOffering `json:"fallback_offering_d,omitempty"` + PlacementOfferings []ExperimentPlacementOffering `json:"placement_offerings,omitempty"` +} + +type ExperimentDurationSettings struct { + ConversionRatePercentage *float64 `json:"conversion_rate_percentage,omitempty"` + DailyEnrolledCustomers *int `json:"daily_enrolled_customers,omitempty"` + MinimumDetectableEffectPercentage *int `json:"minimum_detectable_effect_percentage,omitempty"` + ChanceToWinPercentage *int `json:"chance_to_win_percentage,omitempty"` + ConversionRateIsOverride *bool `json:"conversion_rate_percentage_is_override,omitempty"` + DailyEnrolledIsOverride *bool `json:"daily_enrolled_customers_is_override,omitempty"` +} + +type ExperimentConflict struct { + ID string `json:"id"` + DisplayName string `json:"display_name,omitempty"` +} + +type Experiment struct { + Object string `json:"object"` + ID string `json:"id"` + DisplayName string `json:"display_name"` + Status string `json:"status"` + CreatedAt Millis `json:"created_at"` + UpdatedAt Millis `json:"updated_at"` + StartedAt *Millis `json:"started_at,omitempty"` + StoppedAt *Millis `json:"stopped_at,omitempty"` + PausedAt *Millis `json:"paused_at,omitempty"` + ResumedAt *Millis `json:"resumed_at,omitempty"` + TotalRunningSeconds *int `json:"total_running_time_seconds,omitempty"` + EnrollmentPercent *int `json:"enrollment_percentage,omitempty"` + Priority *int `json:"priority,omitempty"` + OfferingA *ExperimentOffering `json:"offering_a,omitempty"` + OfferingB *ExperimentOffering `json:"offering_b,omitempty"` + OfferingC *ExperimentOffering `json:"offering_c,omitempty"` + OfferingD *ExperimentOffering `json:"offering_d,omitempty"` + Notes *string `json:"notes,omitempty"` + ExperimentType *string `json:"experiment_type,omitempty"` + PrimaryMetric *string `json:"primary_metric,omitempty"` + SecondaryMetrics []string `json:"secondary_metrics,omitempty"` + EnrollmentMode *string `json:"enrollment_mode,omitempty"` + AudienceID *string `json:"audience_id,omitempty"` + TargetingConditions []ExperimentCondition `json:"targeting_conditions,omitempty"` + PinnedAudience any `json:"pinned_audience,omitempty"` + DurationSettings *ExperimentDurationSettings `json:"experiment_duration_settings,omitempty"` + Placements *ExperimentPlacements `json:"placements,omitempty"` + Conflicts []ExperimentConflict `json:"conflicts,omitempty"` } type ListExperimentsOptions struct { diff --git a/internal/cli/experiment_format.go b/internal/cli/experiment_format.go new file mode 100644 index 00000000..fcbe77d7 --- /dev/null +++ b/internal/cli/experiment_format.go @@ -0,0 +1,249 @@ +package cli + +import ( + "fmt" + "strconv" + "strings" + + "github.com/revenuecat/cli/internal/api" + "github.com/revenuecat/cli/internal/output" +) + +func renderExperimentShow(rt *Runtime, experiment *api.Experiment) error { + if rt.Globals.JSON { + return rt.Out.Render(experiment) + } + + variants := []output.CardLine{} + for _, variant := range []struct { + name string + offering *api.ExperimentOffering + }{ + {"Control", experiment.OfferingA}, + {"Treatment", experiment.OfferingB}, + {"Treatment C", experiment.OfferingC}, + {"Treatment D", experiment.OfferingD}, + } { + if variant.offering != nil { + variants = append(variants, output.CardLine{Key: variant.name, Value: experimentOfferingLabel(variant.offering)}) + } + } + + audience := []output.CardLine{} + if experiment.AudienceID != nil && *experiment.AudienceID != "" { + audience = append(audience, output.CardLine{Key: "Audience", Value: *experiment.AudienceID}) + } else if len(experiment.TargetingConditions) > 0 { + for i, condition := range experiment.TargetingConditions { + audience = append(audience, output.CardLine{Key: fmt.Sprintf("Condition %d", i+1), Value: experimentConditionLabel(condition)}) + } + } else { + audience = append(audience, output.CardLine{Key: "Audience", Value: "All eligible customers"}) + } + if experiment.PinnedAudience != nil { + audience = append(audience, output.CardLine{Key: "Pinned", Value: "Audience saved when enrollment started"}) + } + + measurements := []output.CardLine{} + if experiment.EnrollmentPercent != nil { + measurements = append(measurements, output.CardLine{Key: "Enrollment", Value: fmt.Sprintf("%d%%", *experiment.EnrollmentPercent)}) + } + if experiment.EnrollmentMode != nil { + measurements = append(measurements, output.CardLine{Key: "Enrollment mode", Value: *experiment.EnrollmentMode}) + } + if experiment.ExperimentType != nil { + measurements = append(measurements, output.CardLine{Key: "Type", Value: *experiment.ExperimentType}) + } + if experiment.PrimaryMetric != nil { + measurements = append(measurements, output.CardLine{Key: "Primary metric", Value: *experiment.PrimaryMetric}) + } + if len(experiment.SecondaryMetrics) > 0 { + measurements = append(measurements, output.CardLine{Key: "Secondary metrics", Value: strings.Join(experiment.SecondaryMetrics, ", ")}) + } + if experiment.DurationSettings != nil { + measurements = append(measurements, output.CardLine{Key: "Duration settings", Value: experimentDurationSummary(experiment.DurationSettings)}) + } + if experiment.Priority != nil { + measurements = append(measurements, output.CardLine{Key: "Priority", Value: strconv.Itoa(*experiment.Priority)}) + } + if experiment.Notes != nil && *experiment.Notes != "" { + measurements = append(measurements, output.CardLine{Key: "Notes", Value: *experiment.Notes}) + } + + timeline := []output.CardLine{} + for _, event := range []struct { + name string + at *api.Millis + }{ + {"Started", experiment.StartedAt}, + {"Paused", experiment.PausedAt}, + {"Resumed", experiment.ResumedAt}, + {"Stopped", experiment.StoppedAt}, + } { + if event.at != nil { + timeline = append(timeline, output.CardLine{Key: event.name, Value: formatMillis(int64(*event.at))}) + } + } + if experiment.TotalRunningSeconds != nil { + timeline = append(timeline, output.CardLine{Key: "Running time", Value: formatExperimentRunTime(*experiment.TotalRunningSeconds)}) + } + if len(timeline) == 0 { + timeline = append(timeline, output.CardLine{Key: "Created", Value: formatMillis(int64(experiment.CreatedAt))}) + } + + sections := []output.CardSection{ + {Heading: "Variants", Lines: variants}, + {Heading: "Audience", Lines: audience}, + {Heading: "Placements", Lines: experimentPlacementLines(experiment.Placements), Empty: "No placement overrides"}, + {Heading: "Setup", Lines: measurements}, + {Heading: "Timeline", Lines: timeline}, + } + if len(experiment.Conflicts) > 0 { + conflicts := make([]output.CardLine, 0, len(experiment.Conflicts)) + for _, conflict := range experiment.Conflicts { + name := conflict.DisplayName + if name == "" { + name = conflict.ID + } + conflicts = append(conflicts, output.CardLine{Key: conflict.ID, Value: name}) + } + sections = append(sections, output.CardSection{Heading: "Conflicts", Lines: conflicts}) + } + if err := rt.Out.RenderCard(output.Card{ + Title: experiment.DisplayName, + Subtitle: experiment.ID + " · " + experiment.Status, + Sections: sections, + Raw: experiment, + }); err != nil { + return err + } + rt.Out.Hint("Use --json for the complete API response.") + return nil +} + +func experimentOfferingLabel(offering *api.ExperimentOffering) string { + label := offering.ID + if offering.DisplayName != "" { + label += " (" + offering.DisplayName + ")" + } + if offering.PaywallID != "" { + label += " · paywall " + offering.PaywallID + } + return label +} + +func experimentConditionLabel(condition api.ExperimentCondition) string { + field := strings.ReplaceAll(condition.Field, "_", " ") + if condition.Context != nil { + field += " (" + fmt.Sprint(condition.Context) + ")" + } + value := fmt.Sprint(condition.Value) + if values, ok := condition.Value.([]any); ok { + parts := make([]string, len(values)) + for i, item := range values { + parts[i] = fmt.Sprint(item) + } + value = strings.Join(parts, ", ") + } + return field + " " + condition.Operator + " " + value +} + +func experimentConditionSummary(conditions []api.ExperimentCondition) string { + parts := make([]string, len(conditions)) + for i, condition := range conditions { + parts[i] = experimentConditionLabel(condition) + } + return strings.Join(parts, "; ") +} + +func experimentPlacementLines(placements *api.ExperimentPlacements) []output.CardLine { + if placements == nil { + return nil + } + lines := []output.CardLine{} + if fallback := experimentVariantOfferings(placements.FallbackOfferingA, placements.FallbackOfferingB, placements.FallbackOfferingC, placements.FallbackOfferingD); fallback != "" { + lines = append(lines, output.CardLine{Key: "Fallback", Value: fallback}) + } + for _, placement := range placements.PlacementOfferings { + value := experimentVariantOfferings(placement.OfferingA, placement.OfferingB, placement.OfferingC, placement.OfferingD) + if value == "" { + value = "Uses variant defaults" + } + lines = append(lines, output.CardLine{Key: placement.PlacementIdentifier, Value: value}) + } + return lines +} + +func experimentPlacementSummary(placements *api.ExperimentPlacements) string { + lines := experimentPlacementLines(placements) + parts := make([]string, len(lines)) + for i, line := range lines { + parts[i] = line.Key + ": " + line.Value + } + return strings.Join(parts, "; ") +} + +func experimentVariantOfferings(a, b, c, d *api.ExperimentOffering) string { + parts := []string{} + for _, variant := range []struct { + name string + offering *api.ExperimentOffering + }{ + {"Control", a}, {"Treatment", b}, {"Treatment C", c}, {"Treatment D", d}, + } { + if variant.offering != nil { + parts = append(parts, variant.name+" "+experimentOfferingLabel(variant.offering)) + } + } + return strings.Join(parts, " · ") +} + +func experimentDurationSummary(settings *api.ExperimentDurationSettings) string { + parts := []string{} + if settings.ConversionRatePercentage != nil { + parts = append(parts, fmt.Sprintf("expected conversion %.2f%%", *settings.ConversionRatePercentage)) + } + if settings.DailyEnrolledCustomers != nil { + parts = append(parts, fmt.Sprintf("%d customers/day", *settings.DailyEnrolledCustomers)) + } + if settings.MinimumDetectableEffectPercentage != nil { + parts = append(parts, fmt.Sprintf("minimum effect %d%%", *settings.MinimumDetectableEffectPercentage)) + } + if settings.ChanceToWinPercentage != nil { + parts = append(parts, fmt.Sprintf("chance to win %d%%", *settings.ChanceToWinPercentage)) + } + if len(parts) == 0 { + return "Configured" + } + return strings.Join(parts, " · ") +} + +func experimentConflictsSummary(conflicts []api.ExperimentConflict) string { + parts := make([]string, len(conflicts)) + for i, conflict := range conflicts { + parts[i] = conflict.DisplayName + if parts[i] == "" { + parts[i] = conflict.ID + } + } + return strings.Join(parts, ", ") +} + +func formatExperimentRunTime(seconds int) string { + if seconds < 60 { + return fmt.Sprintf("%ds", seconds) + } + days := seconds / 86400 + hours := seconds % 86400 / 3600 + minutes := seconds % 3600 / 60 + parts := []string{} + if days > 0 { + parts = append(parts, fmt.Sprintf("%dd", days)) + } + if hours > 0 || days > 0 { + parts = append(parts, fmt.Sprintf("%dh", hours)) + } + if minutes > 0 || len(parts) == 0 { + parts = append(parts, fmt.Sprintf("%dm", minutes)) + } + return strings.Join(parts, " ") +} diff --git a/internal/cli/experiments.go b/internal/cli/experiments.go index f9ee1f78..65d5e528 100644 --- a/internal/cli/experiments.go +++ b/internal/cli/experiments.go @@ -3,6 +3,7 @@ package cli import ( "fmt" "strconv" + "strings" "github.com/spf13/cobra" @@ -108,13 +109,7 @@ func newExperimentsShowCmd() *cobra.Command { if err != nil { return err } - if err := rt.Out.Render(experiment); err != nil { - return err - } - if !rt.Globals.JSON { - rt.Out.Hint("Use --json to inspect targeting, placements, and all variant details.") - } - return nil + return renderExperimentShow(rt, experiment) }, } } @@ -129,6 +124,17 @@ func newExperimentsResultsCmd() *cobra.Command { Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { rt := RuntimeFrom(cmd.Context()) + var err error + opts.Platform, err = experimentResultsPlatform(opts.Platform) + if err != nil { + return err + } + opts.ExposureStatus, err = experimentExposureStatus(opts.ExposureStatus) + if err != nil { + return err + } + opts.Country = strings.ToUpper(strings.TrimSpace(opts.Country)) + opts.Currency = strings.ToUpper(strings.TrimSpace(opts.Currency)) projectID, err := requireProject(rt) if err != nil { return err @@ -153,39 +159,103 @@ func newExperimentsResultsCmd() *cobra.Command { return renderExperimentResults(rt, results) }, } - cmd.Flags().StringVar(&opts.Platform, "platform", "", "filter by platform") - cmd.Flags().StringVar(&opts.Country, "country", "", "filter by country code") - cmd.Flags().StringVar(&opts.ExposureStatus, "exposure-status", "", "filter by exposure status") + cmd.Flags().StringVar(&opts.Platform, "platform", "", "filter by iOS, Android, macOS, tvOS, watchOS, visionOS, Amazon, Roku, or Web (case-insensitive; app_store/play_store aliases)") + cmd.Flags().StringVar(&opts.Country, "country", "", "filter by ISO country code (for example, US)") + cmd.Flags().StringVar(&opts.ExposureStatus, "exposure-status", "", "filter by exposed or not_exposed; omit for all enrolled customers") cmd.Flags().StringVar(&opts.Currency, "currency", "", "display currency for monetary metrics") return cmd } +func experimentResultsPlatform(value string) (string, error) { + if value == "" { + return "", nil + } + switch strings.ToLower(strings.TrimSpace(value)) { + case "ios", "app_store": + return "iOS", nil + case "android", "play_store": + return "Android", nil + case "macos", "mac_app_store": + return "macOS", nil + case "tvos": + return "tvOS", nil + case "watchos": + return "watchOS", nil + case "visionos": + return "visionOS", nil + case "amazon": + return "Amazon", nil + case "roku": + return "Roku", nil + case "web": + return "Web", nil + default: + return strings.TrimSpace(value), nil + } +} + +func experimentExposureStatus(value string) (string, error) { + if value == "" { + return "", nil + } + switch strings.ToLower(strings.TrimSpace(value)) { + case "exposed": + return "exposed", nil + case "not_exposed": + return "not_exposed", nil + default: + return "", fmt.Errorf("exposure status must be exposed or not_exposed") + } +} + func renderExperimentResults(rt *Runtime, results *api.ExperimentResults) error { rt.Out.Title("Experiment results") rt.Out.Field("Currency", results.Currency) shownSections := 0 for _, section := range results.Sections { - rows := make([][]string, 0) + valuesByMetric := make(map[int]map[string]api.ExperimentResultValue) + hasTreatmentC, hasTreatmentD := false, false for _, value := range section.Values { if value.Metric < 0 || value.Metric >= len(section.Metrics) || value.Segment < 0 || value.Segment >= len(section.Segments) || !section.Segments[value.Segment].IsTotal { continue } - metric := section.Metrics[value.Metric] - shownValue, change := "—", "—" - if value.Value != nil { - shownValue = strconv.FormatFloat(*value.Value, 'f', -1, 64) + if valuesByMetric[value.Metric] == nil { + valuesByMetric[value.Metric] = make(map[string]api.ExperimentResultValue) } - if value.Change != nil { - change = fmt.Sprintf("%+.2f%%", *value.Change) - } - rows = append(rows, []string{metric.Name, value.Variant, shownValue, metric.Unit, change}) + valuesByMetric[value.Metric][value.Variant] = value + hasTreatmentC = hasTreatmentC || value.Variant == "TreatmentC" + hasTreatmentD = hasTreatmentD || value.Variant == "TreatmentD" } - if len(rows) == 0 { + if len(valuesByMetric) == 0 { continue } + columns := []string{"METRIC", "CONTROL", "TREATMENT", "CHANGE"} + if hasTreatmentC { + columns = append(columns, "TREATMENT C", "CHANGE C") + } + if hasTreatmentD { + columns = append(columns, "TREATMENT D", "CHANGE D") + } + columns = append(columns, "UNIT") + rows := make([][]string, 0, len(valuesByMetric)) + for i, metric := range section.Metrics { + values := valuesByMetric[i] + if len(values) == 0 { + continue + } + row := []string{metric.Name, resultValue(values["Control"]), resultValue(values["Treatment"]), resultChange(values["Treatment"])} + if hasTreatmentC { + row = append(row, resultValue(values["TreatmentC"]), resultChange(values["TreatmentC"])) + } + if hasTreatmentD { + row = append(row, resultValue(values["TreatmentD"]), resultChange(values["TreatmentD"])) + } + row = append(row, metric.Unit) + rows = append(rows, row) + } shownSections++ rt.Out.Title(section.Section) - if err := rt.Out.RenderTable(output.Table{Columns: []string{"METRIC", "VARIANT", "VALUE", "UNIT", "CHANGE"}, Rows: rows}); err != nil { + if err := rt.Out.RenderTable(output.Table{Columns: columns, Rows: rows}); err != nil { return err } } @@ -203,3 +273,17 @@ func renderExperimentResults(rt *Runtime, results *api.ExperimentResults) error } return nil } + +func resultValue(value api.ExperimentResultValue) string { + if value.Value == nil { + return "—" + } + return strconv.FormatFloat(*value.Value, 'f', -1, 64) +} + +func resultChange(value api.ExperimentResultValue) string { + if value.Change == nil { + return "—" + } + return fmt.Sprintf("%+.2f%%", *value.Change) +} diff --git a/internal/cli/experiments_test.go b/internal/cli/experiments_test.go index 03472d2c..1484beae 100644 --- a/internal/cli/experiments_test.go +++ b/internal/cli/experiments_test.go @@ -41,6 +41,41 @@ func TestExperimentResultsJSONIncludesAllSegments(t *testing.T) { } } +func TestExperimentResultsNormalizesFilters(t *testing.T) { + for _, tc := range []struct { + platform string + want string + }{ + {"ios", "iOS"}, + {"app_store", "iOS"}, + {"iOS", "iOS"}, + {"android", "Android"}, + } { + t.Run(tc.platform, func(t *testing.T) { + var platform, exposure string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + platform = r.URL.Query().Get("platform") + exposure = r.URL.Query().Get("exposure_status") + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[],"statistics":[],"predicted_ltv":null}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + _, _, err := runAgentCmd(t, "experiments", "results", "exp1", "--platform", tc.platform, "--exposure-status", "EXPOSED", "--project-id", "proj", "--api-key", "sk_test", "--json", "--no-input") + if err != nil || platform != tc.want || exposure != "exposed" { + t.Fatalf("err=%v platform=%q exposure=%q", err, platform, exposure) + } + }) + } +} + +func TestExperimentResultsRejectsUnknownExposureStatus(t *testing.T) { + _, _, err := runCmd(t, "experiments", "results", "exp1", "--exposure-status", "viewed") + if err == nil || !strings.Contains(err.Error(), "exposure status must be") { + t.Fatalf("expected supported-value error, got %v", err) + } +} + func TestExperimentResultsHumanExplainsMissingTotals(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index 22f9b361..18deeb7f 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -28,6 +28,8 @@ func snapshotServer(t *testing.T) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { + case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap"): + io.WriteString(w, `{"object":"experiment","id":"exp_snap","display_name":"Price — paywall","status":"paused","created_at":1784297950368,"updated_at":1784297950368,"total_running_time_seconds":1989824,"enrollment_percentage":50,"offering_a":{"id":"ofrng_control","display_name":"Control"},"offering_b":{"id":"ofrng_treatment","display_name":"Treatment"},"targeting_conditions":[{"field":"platform","operator":"in","value":["ios"]}],"placements":{"placement_offerings":[{"placement_identifier":"onboarding","offering_a":{"id":"ofrng_control"},"offering_b":{"id":"ofrng_treatment"}}]},"primary_metric":"initial_conversion_rate","secondary_metrics":["realized_ltv_per_customer","trial_conversion_rate"],"experiment_duration_settings":{"chance_to_win_percentage":95}}`) case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap/results"): io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[{"object":"experiment_results_section","section":"Conversion","metrics":[{"name":"trial_conversion_rate","unit":"%"}],"segments":[{"object":"experiment_results_segment","id":"total","display_name":"Total","is_total":true}],"values":[{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Control","value":12.5,"change":null,"credible_interval":null,"lift_credible_interval":null},{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Treatment","value":15.0,"change":20,"credible_interval":null,"lift_credible_interval":null}]}],"statistics":[],"predicted_ltv":null}`) case strings.HasSuffix(r.URL.Path, "/offerings/ofrng_snap"): @@ -60,6 +62,7 @@ func TestOutputSnapshots(t *testing.T) { {"version", []string{"version"}}, {"auth-status-logged-out", []string{"auth", "status", "--no-input"}}, {"offerings-show", []string{"offerings", "show", "ofrng_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, + {"experiments-show", []string{"experiments", "show", "exp_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"experiments-results", []string{"experiments", "results", "exp_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"apps-list", []string{"apps", "list", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"apps-list-all-projects", []string{"apps", "list", "--all-projects", "--bundle-id", "com.example.moodly", "--no-input", "--api-key", "sk_snap"}}, diff --git a/internal/cli/testdata/snapshots/apps-list-all-projects.golden b/internal/cli/testdata/snapshots/apps-list-all-projects.golden index cc733a02..4125fb27 100644 --- a/internal/cli/testdata/snapshots/apps-list-all-projects.golden +++ b/internal/cli/testdata/snapshots/apps-list-all-projects.golden @@ -1,6 +1,6 @@ $ rc apps list --all-projects --bundle-id com.example.moodly --no-input --api-key sk_snap ! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. Fix it: rc setup apple app_snap1 --project-id proj_snap (interactive Apple sign-in with 2FA) -PROJECT ID NAME TYPE CREATED CREDENTIALS +PROJECT ID NAME TYPE CREATED CREDENTIALS proj_snap app_snap1 Moodly (App Store) app_store 2026-07-16 partial — run: rc setup apple -proj_snap2 app_snap3 Moodly Staging (App Store) app_store 2026-07-16 ready +proj_snap2 app_snap3 Moodly Staging (App Store) app_store 2026-07-16 ready diff --git a/internal/cli/testdata/snapshots/apps-list.golden b/internal/cli/testdata/snapshots/apps-list.golden index 1bc23115..ba8db20b 100644 --- a/internal/cli/testdata/snapshots/apps-list.golden +++ b/internal/cli/testdata/snapshots/apps-list.golden @@ -1,6 +1,6 @@ $ rc apps list --no-input --project-id proj_snap --api-key sk_snap ! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. Fix it: rc setup apple app_snap1 (interactive Apple sign-in with 2FA) -ID NAME TYPE CREATED CREDENTIALS +ID NAME TYPE CREATED CREDENTIALS app_snap1 Moodly (App Store) app_store 2026-07-16 partial — run: rc setup apple app_snap2 Test Store test_store 2026-07-16 — diff --git a/internal/cli/testdata/snapshots/experiments-results.golden b/internal/cli/testdata/snapshots/experiments-results.golden index a1a761b0..988c34e7 100644 --- a/internal/cli/testdata/snapshots/experiments-results.golden +++ b/internal/cli/testdata/snapshots/experiments-results.golden @@ -4,6 +4,5 @@ $ rc experiments results exp_snap --no-input --project-id proj_snap --api-key sk Currency USD ▍ Conversion -METRIC VARIANT VALUE UNIT CHANGE -trial_conversion_rate Control 12.5 % — -trial_conversion_rate Treatment 15 % +20.00% +METRIC CONTROL TREATMENT CHANGE UNIT +trial_conversion_rate 12.5 15 +20.00% % diff --git a/internal/cli/testdata/snapshots/experiments-show.golden b/internal/cli/testdata/snapshots/experiments-show.golden new file mode 100644 index 00000000..b4492560 --- /dev/null +++ b/internal/cli/testdata/snapshots/experiments-show.golden @@ -0,0 +1,23 @@ +$ rc experiments show exp_snap --no-input --project-id proj_snap --api-key sk_snap + Use --json for the complete API response. +▍ Price — paywall + exp_snap · paused + +Variants + Control: ofrng_control (Control) + Treatment: ofrng_treatment (Treatment) + +Audience + Condition 1: platform in ios + +Placements + onboarding: Control ofrng_control · Treatment ofrng_treatment + +Setup + Enrollment: 50% + Primary metric: initial_conversion_rate + Secondary metrics: realized_ltv_per_customer, trial_conversion_rate + Duration settings: chance to win 95% + +Timeline + Running time: 23d 0h 43m diff --git a/internal/output/card.go b/internal/output/card.go index 3e6c86c7..538122b2 100644 --- a/internal/output/card.go +++ b/internal/output/card.go @@ -143,14 +143,14 @@ func (r *Renderer) writeCardTable(t CardTable) { } widths := make([]int, len(t.Columns)) for i, c := range t.Columns { - widths[i] = len(c) + widths[i] = lipgloss.Width(c) } for _, row := range rows { for i, cell := range row { if i >= len(widths) { continue } - if l := len(cell); l > widths[i] { + if l := lipgloss.Width(cell); l > widths[i] { widths[i] = l } } @@ -179,8 +179,8 @@ func (r *Renderer) writeCardTable(t CardTable) { func (r *Renderer) writeLines(lines []CardLine) { keyWidth := 0 for _, l := range lines { - if k := len(SanitizeLine(l.Key)); k > keyWidth { - keyWidth = k + if width := lipgloss.Width(SanitizeLine(l.Key)); width > keyWidth { + keyWidth = width } } keyStyle := lipgloss.NewStyle().Faint(true) diff --git a/internal/output/output.go b/internal/output/output.go index e015fa32..9ff7d32d 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -369,14 +369,14 @@ func (r *Renderer) RenderTable(t Table) error { } widths := make([]int, len(t.Columns)) for i, c := range t.Columns { - widths[i] = len(c) + widths[i] = lipgloss.Width(c) } for _, row := range rows { for i, cell := range row { if i >= len(widths) { continue } - if l := len(cell); l > widths[i] { + if l := lipgloss.Width(cell); l > widths[i] { widths[i] = l } } @@ -402,10 +402,11 @@ func (r *Renderer) RenderTable(t Table) error { } func padRight(s string, n int) string { - if len(s) >= n { + width := lipgloss.Width(s) + if width >= n { return s } - return s + spaces(n-len(s)) + return s + spaces(n-width) } func spaces(n int) string { diff --git a/internal/output/output_test.go b/internal/output/output_test.go index 32588e5b..f0a19ac8 100644 --- a/internal/output/output_test.go +++ b/internal/output/output_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "github.com/charmbracelet/lipgloss" + "github.com/revenuecat/cli/internal/output" ) @@ -90,6 +92,33 @@ func TestRenderTable_AlignsColumnsByWidestRow(t *testing.T) { } } +func TestRenderTable_AlignsUnicodeByDisplayWidth(t *testing.T) { + r, out, _ := newR(false) + err := r.RenderTable(output.Table{ + Columns: []string{"NAME", "STATUS"}, + Rows: [][]string{ + {"Trial — iOS", "running"}, + {"Test", "draft"}, + }, + }) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + for _, line := range lines { + status := strings.Index(line, "STATUS") + if status < 0 { + status = strings.Index(line, "running") + } + if status < 0 { + status = strings.Index(line, "draft") + } + if status < 0 || lipgloss.Width(line[:status]) != 13 { + t.Fatalf("second column shifted: %q", line) + } + } +} + func TestRenderTable_JSONMode_EmitsRawNotTable(t *testing.T) { r, out, _ := newR(true) raw := map[string]any{"items": []any{map[string]any{"id": "x"}}} From a752e0ae0a6799a41c93b74a55b01be6fee97b32 Mon Sep 17 00:00:00 2001 From: Josh Holtz Date: Fri, 25 Sep 2026 14:10:37 -0500 Subject: [PATCH 4/7] fix(experiments): warn on unknown platforms and order timeline --- docs/command-surface.md | 4 ++-- internal/cli/experiment_format.go | 12 +++++++++- internal/cli/experiments.go | 38 ++++++++++++++++--------------- internal/cli/experiments_test.go | 29 +++++++++++++++++++++++ 4 files changed, 62 insertions(+), 21 deletions(-) diff --git a/docs/command-surface.md b/docs/command-surface.md index 18089165..fa79daba 100644 --- a/docs/command-surface.md +++ b/docs/command-surface.md @@ -200,8 +200,8 @@ rc paywalls delete [--force] # attached/published pa rc flows list [--type paywall|web-funnel|general] [--exclude-type ...] # experimental (hidden from --help until Checkpoint rules roll out); repeatable; shows type and publish state # Experiments — development v2 endpoints, normal discoverable CLI surface rc experiments list [--status ] -rc experiments show # human detail view shows variant offerings, targeting, placements, and readable run time -rc experiments results [--platform

] [--country ] [--exposure-status exposed|not_exposed] [--currency ] # human totals compare variants by metric +rc experiments show # human detail view shows variants, targeting, placements, and chronological timeline +rc experiments results [--platform

] [--country ] [--exposure-status exposed|not_exposed] [--currency ] # human totals compare variants by metric; unknown platforms warn before passing through # Rico (AI assistant) rc rico [message] # streaming chat window in a TTY (--plain for a line loop) diff --git a/internal/cli/experiment_format.go b/internal/cli/experiment_format.go index fcbe77d7..24a2ea91 100644 --- a/internal/cli/experiment_format.go +++ b/internal/cli/experiment_format.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "sort" "strconv" "strings" @@ -70,6 +71,11 @@ func renderExperimentShow(rt *Runtime, experiment *api.Experiment) error { } timeline := []output.CardLine{} + type timelineEvent struct { + name string + at api.Millis + } + events := []timelineEvent{} for _, event := range []struct { name string at *api.Millis @@ -80,9 +86,13 @@ func renderExperimentShow(rt *Runtime, experiment *api.Experiment) error { {"Stopped", experiment.StoppedAt}, } { if event.at != nil { - timeline = append(timeline, output.CardLine{Key: event.name, Value: formatMillis(int64(*event.at))}) + events = append(events, timelineEvent{name: event.name, at: *event.at}) } } + sort.SliceStable(events, func(i, j int) bool { return events[i].at < events[j].at }) + for _, event := range events { + timeline = append(timeline, output.CardLine{Key: event.name, Value: formatMillis(int64(event.at))}) + } if experiment.TotalRunningSeconds != nil { timeline = append(timeline, output.CardLine{Key: "Running time", Value: formatExperimentRunTime(*experiment.TotalRunningSeconds)}) } diff --git a/internal/cli/experiments.go b/internal/cli/experiments.go index 65d5e528..99aa70cc 100644 --- a/internal/cli/experiments.go +++ b/internal/cli/experiments.go @@ -124,11 +124,12 @@ func newExperimentsResultsCmd() *cobra.Command { Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { rt := RuntimeFrom(cmd.Context()) - var err error - opts.Platform, err = experimentResultsPlatform(opts.Platform) - if err != nil { - return err + platform, knownPlatform := experimentResultsPlatform(opts.Platform) + opts.Platform = platform + if !knownPlatform { + rt.Out.AlwaysWarn(fmt.Sprintf("Unknown platform %q; sending it as provided. Results may be empty.", platform)) } + var err error opts.ExposureStatus, err = experimentExposureStatus(opts.ExposureStatus) if err != nil { return err @@ -159,38 +160,39 @@ func newExperimentsResultsCmd() *cobra.Command { return renderExperimentResults(rt, results) }, } - cmd.Flags().StringVar(&opts.Platform, "platform", "", "filter by iOS, Android, macOS, tvOS, watchOS, visionOS, Amazon, Roku, or Web (case-insensitive; app_store/play_store aliases)") + cmd.Flags().StringVar(&opts.Platform, "platform", "", "filter by iOS, Android, macOS, tvOS, watchOS, visionOS, Amazon, Roku, or Web (case-insensitive; app_store/play_store aliases); other values pass through with a warning") cmd.Flags().StringVar(&opts.Country, "country", "", "filter by ISO country code (for example, US)") cmd.Flags().StringVar(&opts.ExposureStatus, "exposure-status", "", "filter by exposed or not_exposed; omit for all enrolled customers") cmd.Flags().StringVar(&opts.Currency, "currency", "", "display currency for monetary metrics") return cmd } -func experimentResultsPlatform(value string) (string, error) { +func experimentResultsPlatform(value string) (string, bool) { + value = strings.TrimSpace(value) if value == "" { - return "", nil + return "", true } - switch strings.ToLower(strings.TrimSpace(value)) { + switch strings.ToLower(value) { case "ios", "app_store": - return "iOS", nil + return "iOS", true case "android", "play_store": - return "Android", nil + return "Android", true case "macos", "mac_app_store": - return "macOS", nil + return "macOS", true case "tvos": - return "tvOS", nil + return "tvOS", true case "watchos": - return "watchOS", nil + return "watchOS", true case "visionos": - return "visionOS", nil + return "visionOS", true case "amazon": - return "Amazon", nil + return "Amazon", true case "roku": - return "Roku", nil + return "Roku", true case "web": - return "Web", nil + return "Web", true default: - return strings.TrimSpace(value), nil + return value, false } } diff --git a/internal/cli/experiments_test.go b/internal/cli/experiments_test.go index 1484beae..fcfaf633 100644 --- a/internal/cli/experiments_test.go +++ b/internal/cli/experiments_test.go @@ -76,6 +76,35 @@ func TestExperimentResultsRejectsUnknownExposureStatus(t *testing.T) { } } +func TestExperimentResultsWarnsOnUnknownPlatform(t *testing.T) { + var platform string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + platform = r.URL.Query().Get("platform") + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[],"statistics":[],"predicted_ltv":null}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + _, stderr, err := runAgentCmd(t, "experiments", "results", "exp1", "--platform", "bogus", "--project-id", "proj", "--api-key", "sk_test", "--no-input") + if err != nil || platform != "bogus" || !strings.Contains(stderr, `Unknown platform "bogus"`) { + t.Fatalf("err=%v platform=%q stderr=%q", err, platform, stderr) + } +} + +func TestExperimentShowSortsTimeline(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Timeline test","status":"paused","started_at":1000,"paused_at":3000,"resumed_at":2000}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + out, _, err := runAgentCmd(t, "experiments", "show", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input") + started, resumed, paused := strings.Index(out, "Started:"), strings.Index(out, "Resumed:"), strings.Index(out, "Paused:") + if err != nil || started < 0 || resumed < 0 || paused < 0 || started > resumed || resumed > paused { + t.Fatalf("timeline out of order: err=%v out=%s", err, out) + } +} + func TestExperimentResultsHumanExplainsMissingTotals(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") From feb8f7e3a38d189e0168bc816d4888bb2d7a4e6d Mon Sep 17 00:00:00 2001 From: Josh Holtz Date: Fri, 25 Sep 2026 14:49:20 -0500 Subject: [PATCH 5/7] feat(experiments): show decision signals in results --- docs/command-surface.md | 2 +- docs/previews/experiments-results.svg | 12 +++-- internal/api/experiments.go | 20 +++++++-- internal/cli/experiments.go | 45 +++++++++++++++++++ internal/cli/snapshot_test.go | 2 +- .../snapshots/experiments-results.golden | 4 ++ 6 files changed, 76 insertions(+), 9 deletions(-) diff --git a/docs/command-surface.md b/docs/command-surface.md index fa79daba..4740553b 100644 --- a/docs/command-surface.md +++ b/docs/command-surface.md @@ -201,7 +201,7 @@ rc flows list [--type paywall|web-funnel|general] [--exclude-type ...] # exper # Experiments — development v2 endpoints, normal discoverable CLI surface rc experiments list [--status ] rc experiments show # human detail view shows variants, targeting, placements, and chronological timeline -rc experiments results [--platform

] [--country ] [--exposure-status exposed|not_exposed] [--currency ] # human totals compare variants by metric; unknown platforms warn before passing through +rc experiments results [--platform

] [--country ] [--exposure-status exposed|not_exposed] [--currency ] # human totals and decision signals; unknown platforms warn before passing through # Rico (AI assistant) rc rico [message] # streaming chat window in a TTY (--plain for a line loop) diff --git a/docs/previews/experiments-results.svg b/docs/previews/experiments-results.svg index ecf310d0..91ddde37 100644 --- a/docs/previews/experiments-results.svg +++ b/docs/previews/experiments-results.svg @@ -1,6 +1,6 @@ - + + $ rc experiments results exp_snap --no-input --project-id proj_snap --api-key sk_snap @@ -16,8 +16,12 @@   Currency                    USD ▍ Conversion -METRIC                 CONTROL  TREATMENT  CHANGE   UNIT -trial_conversion_rate  12.5     15         +20.00%  %    + +▍ Decision signals +METRIC                 CONTROL  TREATMENT  CHANGE   UNIT +trial_conversion_rate  12.5     15         +20.00%  %    +METRIC                 VARIANT    CHANCE TO WIN  95% LIFT INTERVAL +trial_conversion_rate  Treatment  98.0%          -5.0% to +22.0%   diff --git a/internal/api/experiments.go b/internal/api/experiments.go index 23a3a52a..58ffea07 100644 --- a/internal/api/experiments.go +++ b/internal/api/experiments.go @@ -2,6 +2,7 @@ package api import ( "context" + "encoding/json" "net/http" "net/url" "strconv" @@ -158,9 +159,22 @@ type ExperimentResultValue struct { } type ExperimentResultStatistic struct { - Object string `json:"object"` - MetricName string `json:"metric_name"` - Variants []any `json:"variants"` + Object string `json:"object"` + MetricName string `json:"metric_name"` + Variants []json.RawMessage `json:"variants"` +} + +type ExperimentResultVariantStatistic struct { + Object string `json:"object"` + Name string `json:"name"` + ChanceToWin *float64 `json:"chance_to_win"` + ChanceToWinStatus string `json:"chance_to_win_status"` + CredibleIntervalLower *float64 `json:"credible_interval_lower"` + CredibleIntervalUpper *float64 `json:"credible_interval_upper"` + CredibleIntervalStatus string `json:"credible_interval_status"` + LiftCredibleIntervalLower *float64 `json:"lift_credible_interval_lower"` + LiftCredibleIntervalUpper *float64 `json:"lift_credible_interval_upper"` + LiftCredibleIntervalStatus string `json:"lift_credible_interval_status"` } type ExperimentPredictedLTV struct { diff --git a/internal/cli/experiments.go b/internal/cli/experiments.go index 99aa70cc..3fba5887 100644 --- a/internal/cli/experiments.go +++ b/internal/cli/experiments.go @@ -1,6 +1,7 @@ package cli import ( + "encoding/json" "fmt" "strconv" "strings" @@ -265,6 +266,30 @@ func renderExperimentResults(rt *Runtime, results *api.ExperimentResults) error rt.Out.Info("No total-segment metrics available.") rt.Out.Hint("Use --json to inspect all segments and statistics.") } + statistics := [][]string{} + for _, metric := range results.Statistics { + for _, raw := range metric.Variants { + var variant api.ExperimentResultVariantStatistic + if err := json.Unmarshal(raw, &variant); err != nil { + return err + } + if variant.Name == "Control" { + continue + } + statistics = append(statistics, []string{ + metric.MetricName, + variant.Name, + chanceToWinLabel(variant), + liftIntervalLabel(variant), + }) + } + } + if len(statistics) > 0 { + rt.Out.Title("Decision signals") + if err := rt.Out.RenderTable(output.Table{Columns: []string{"METRIC", "VARIANT", "CHANCE TO WIN", "95% LIFT INTERVAL"}, Rows: statistics}); err != nil { + return err + } + } if results.PredictedLTV != nil { winner := map[string]string{"a": "Control", "b": "Treatment", "c": "Treatment C", "d": "Treatment D"}[results.PredictedLTV.PredictedWinnerVariant] if winner == "" { @@ -276,6 +301,26 @@ func renderExperimentResults(rt *Runtime, results *api.ExperimentResults) error return nil } +func chanceToWinLabel(stat api.ExperimentResultVariantStatistic) string { + if stat.ChanceToWin != nil { + return fmt.Sprintf("%.1f%%", *stat.ChanceToWin*100) + } + if stat.ChanceToWinStatus == "insufficient_data" { + return "Need more data" + } + return "—" +} + +func liftIntervalLabel(stat api.ExperimentResultVariantStatistic) string { + if stat.LiftCredibleIntervalLower != nil && stat.LiftCredibleIntervalUpper != nil { + return fmt.Sprintf("%+.1f%% to %+.1f%%", *stat.LiftCredibleIntervalLower*100, *stat.LiftCredibleIntervalUpper*100) + } + if stat.LiftCredibleIntervalStatus == "insufficient_data" { + return "Need more data" + } + return "—" +} + func resultValue(value api.ExperimentResultValue) string { if value.Value == nil { return "—" diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index 18deeb7f..f9e8394f 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -31,7 +31,7 @@ func snapshotServer(t *testing.T) *httptest.Server { case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap"): io.WriteString(w, `{"object":"experiment","id":"exp_snap","display_name":"Price — paywall","status":"paused","created_at":1784297950368,"updated_at":1784297950368,"total_running_time_seconds":1989824,"enrollment_percentage":50,"offering_a":{"id":"ofrng_control","display_name":"Control"},"offering_b":{"id":"ofrng_treatment","display_name":"Treatment"},"targeting_conditions":[{"field":"platform","operator":"in","value":["ios"]}],"placements":{"placement_offerings":[{"placement_identifier":"onboarding","offering_a":{"id":"ofrng_control"},"offering_b":{"id":"ofrng_treatment"}}]},"primary_metric":"initial_conversion_rate","secondary_metrics":["realized_ltv_per_customer","trial_conversion_rate"],"experiment_duration_settings":{"chance_to_win_percentage":95}}`) case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap/results"): - io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[{"object":"experiment_results_section","section":"Conversion","metrics":[{"name":"trial_conversion_rate","unit":"%"}],"segments":[{"object":"experiment_results_segment","id":"total","display_name":"Total","is_total":true}],"values":[{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Control","value":12.5,"change":null,"credible_interval":null,"lift_credible_interval":null},{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Treatment","value":15.0,"change":20,"credible_interval":null,"lift_credible_interval":null}]}],"statistics":[],"predicted_ltv":null}`) + io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[{"object":"experiment_results_section","section":"Conversion","metrics":[{"name":"trial_conversion_rate","unit":"%"}],"segments":[{"object":"experiment_results_segment","id":"total","display_name":"Total","is_total":true}],"values":[{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Control","value":12.5,"change":null,"credible_interval":null,"lift_credible_interval":null},{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Treatment","value":15.0,"change":20,"credible_interval":null,"lift_credible_interval":null}]}],"statistics":[{"object":"experiment_results_statistic","metric_name":"trial_conversion_rate","variants":[{"object":"experiment_results_variant_statistic","name":"Control","chance_to_win":null,"chance_to_win_status":"unavailable","credible_interval_lower":0.1,"credible_interval_upper":0.15,"credible_interval_status":"ok","lift_credible_interval_lower":null,"lift_credible_interval_upper":null,"lift_credible_interval_status":"unavailable"},{"object":"experiment_results_variant_statistic","name":"Treatment","chance_to_win":0.98,"chance_to_win_status":"ok","credible_interval_lower":0.11,"credible_interval_upper":0.18,"credible_interval_status":"ok","lift_credible_interval_lower":-0.05,"lift_credible_interval_upper":0.22,"lift_credible_interval_status":"ok"}]}],"predicted_ltv":null}`) case strings.HasSuffix(r.URL.Path, "/offerings/ofrng_snap"): io.WriteString(w, `{"object":"offering","id":"ofrng_snap","lookup_key":"default","display_name":"Default","is_current":true,"created_at":1784297950368,"project_id":"proj_snap"}`) case strings.HasSuffix(r.URL.Path, "/apps/app_snap1"): diff --git a/internal/cli/testdata/snapshots/experiments-results.golden b/internal/cli/testdata/snapshots/experiments-results.golden index 988c34e7..f3c1052f 100644 --- a/internal/cli/testdata/snapshots/experiments-results.golden +++ b/internal/cli/testdata/snapshots/experiments-results.golden @@ -4,5 +4,9 @@ $ rc experiments results exp_snap --no-input --project-id proj_snap --api-key sk Currency USD ▍ Conversion + +▍ Decision signals METRIC CONTROL TREATMENT CHANGE UNIT trial_conversion_rate 12.5 15 +20.00% % +METRIC VARIANT CHANCE TO WIN 95% LIFT INTERVAL +trial_conversion_rate Treatment 98.0% -5.0% to +22.0% From 33634249e02e5e72dc26c3abe78b7d1eb7b314bd Mon Sep 17 00:00:00 2001 From: Josh Holtz Date: Tue, 6 Oct 2026 16:35:15 -0500 Subject: [PATCH 6/7] fix(experiments): correct result units and complete read output --- docs/previews/apps-list-all-projects.svg | 10 +-- docs/previews/apps-list.svg | 10 +-- docs/previews/experiments-list.svg | 18 ++++ docs/previews/experiments-results.svg | 8 +- docs/previews/experiments-show.svg | 44 ++++----- internal/api/experiments.go | 7 ++ internal/api/experiments_test.go | 4 + internal/cli/experiment_format.go | 7 ++ internal/cli/experiments.go | 67 ++++++++++---- internal/cli/experiments_picker_test.go | 39 ++++++++ internal/cli/experiments_test.go | 90 +++++++++++++++++++ internal/cli/snapshot_test.go | 22 +++-- .../snapshots/apps-list-all-projects.golden | 4 +- .../cli/testdata/snapshots/apps-list.golden | 4 +- .../snapshots/experiments-list.golden | 3 + .../snapshots/experiments-results.golden | 4 +- .../snapshots/experiments-show.golden | 2 +- 17 files changed, 273 insertions(+), 70 deletions(-) create mode 100644 docs/previews/experiments-list.svg create mode 100644 internal/cli/experiments_picker_test.go create mode 100644 internal/cli/testdata/snapshots/experiments-list.golden diff --git a/docs/previews/apps-list-all-projects.svg b/docs/previews/apps-list-all-projects.svg index 50c41aa5..23e917b4 100644 --- a/docs/previews/apps-list-all-projects.svg +++ b/docs/previews/apps-list-all-projects.svg @@ -11,11 +11,11 @@ $ rc apps list --all-projects --bundle-id com.example.moodly --no-input --api-key sk_snap -! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. -  Fix it:  rc setup apple app_snap1 --project-id proj_snap  (interactive Apple sign-in with 2FA) -PROJECT     ID         NAME                        TYPE       CREATED     CREDENTIALS                   -proj_snap   app_snap1  Moodly (App Store)          app_store  2026-07-16  partial — run: rc setup apple -proj_snap2  app_snap3  Moodly Staging (App Store)  app_store  2026-07-16  ready                         +PROJECT     ID         NAME                        TYPE       CREATED     CREDENTIALS                   +proj_snap   app_snap1  Moodly (App Store)          app_store  2026-07-16  partial — run: rc setup apple +proj_snap2  app_snap3  Moodly Staging (App Store)  app_store  2026-07-16  ready                         +! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. +  Fix it:  rc setup apple app_snap1 --project-id proj_snap  (interactive Apple sign-in with 2FA) diff --git a/docs/previews/apps-list.svg b/docs/previews/apps-list.svg index 5c092486..50c5ee02 100644 --- a/docs/previews/apps-list.svg +++ b/docs/previews/apps-list.svg @@ -11,11 +11,11 @@ $ rc apps list --no-input --project-id proj_snap --api-key sk_snap -! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. -  Fix it:  rc setup apple app_snap1  (interactive Apple sign-in with 2FA) -ID         NAME                TYPE        CREATED     CREDENTIALS                   -app_snap1  Moodly (App Store)  app_store   2026-07-16  partial — run: rc setup apple -app_snap2  Test Store          test_store  2026-07-16  —                             +ID         NAME                TYPE        CREATED     CREDENTIALS                   +app_snap1  Moodly (App Store)  app_store   2026-07-16  partial — run: rc setup apple +app_snap2  Test Store          test_store  2026-07-16  —                             +! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. +  Fix it:  rc setup apple app_snap1  (interactive Apple sign-in with 2FA) diff --git a/docs/previews/experiments-list.svg b/docs/previews/experiments-list.svg new file mode 100644 index 00000000..20497d68 --- /dev/null +++ b/docs/previews/experiments-list.svg @@ -0,0 +1,18 @@ + + + + + +$ rc experiments list --no-input --project-id proj_snap --api-key sk_snap +ID        NAME           STATUS  CONTROL  TREATMENT  TREATMENT C  TREATMENT D +exp_snap  Four variants  draft   ofrng_a  ofrng_b    ofrng_c      ofrng_d     + + + diff --git a/docs/previews/experiments-results.svg b/docs/previews/experiments-results.svg index 91ddde37..142a3c17 100644 --- a/docs/previews/experiments-results.svg +++ b/docs/previews/experiments-results.svg @@ -16,10 +16,10 @@   Currency                    USD ▍ Conversion - -▍ Decision signals -METRIC                 CONTROL  TREATMENT  CHANGE   UNIT -trial_conversion_rate  12.5     15         +20.00%  %    +METRIC                 CONTROL  TREATMENT  CHANGE   UNIT +trial_conversion_rate  12.5     15         +20.00%  %    + +▍ Decision signals METRIC                 VARIANT    CHANCE TO WIN  95% LIFT INTERVAL trial_conversion_rate  Treatment  98.0%          -5.0% to +22.0%   diff --git a/docs/previews/experiments-show.svg b/docs/previews/experiments-show.svg index 882b24e0..16bba185 100644 --- a/docs/previews/experiments-show.svg +++ b/docs/previews/experiments-show.svg @@ -11,28 +11,28 @@ $ rc experiments show exp_snap --no-input --project-id proj_snap --api-key sk_snap -  Use --json for the complete API response. -▍ Price — paywall -  exp_snap · paused - -Variants -  Control:    ofrng_control (Control) -  Treatment:  ofrng_treatment (Treatment) - -Audience -  Condition 1:  platform in ios - -Placements -  onboarding:  Control ofrng_control · Treatment ofrng_treatment - -Setup -  Enrollment:         50% -  Primary metric:     initial_conversion_rate -  Secondary metrics:  realized_ltv_per_customer, trial_conversion_rate -  Duration settings:  chance to win 95% - -Timeline -  Running time:  23d 0h 43m +▍ Price — paywall +  exp_snap · paused + +Variants +  Control:    ofrng_control (Control) +  Treatment:  ofrng_treatment (Treatment) + +Audience +  Condition 1:  platform in ios + +Placements +  onboarding:  Control ofrng_control · Treatment ofrng_treatment + +Setup +  Enrollment:         50% +  Primary metric:     initial_conversion_rate +  Secondary metrics:  realized_ltv_per_customer, trial_conversion_rate +  Duration settings:  chance to win 95% + +Timeline +  Running time:  23d 0h 43m +  Use --json for the complete API response. diff --git a/internal/api/experiments.go b/internal/api/experiments.go index 58ffea07..ee82b3c3 100644 --- a/internal/api/experiments.go +++ b/internal/api/experiments.go @@ -117,6 +117,13 @@ func (s *ExperimentsService) Get(ctx context.Context, projectID, id string) (*Ex return &out, err } +func (s *ExperimentsService) GetWithPaywalls(ctx context.Context, projectID, id string) (*Experiment, error) { + var out Experiment + path := pathExperiment(projectID, id) + "?expand=offering.paywall" + err := s.c.do(ctx, http.MethodGet, path, nil, &out) + return &out, err +} + type ExperimentResults struct { Object string `json:"object"` Sections []ExperimentResultsSection `json:"sections"` diff --git a/internal/api/experiments_test.go b/internal/api/experiments_test.go index 430268ee..6df2b4a6 100644 --- a/internal/api/experiments_test.go +++ b/internal/api/experiments_test.go @@ -39,6 +39,9 @@ func TestExperimentsReadRoutesAndFilters(t *testing.T) { if _, err := client.Experiments.Get(ctx, "proj", "exp1"); err != nil { t.Fatal(err) } + if _, err := client.Experiments.GetWithPaywalls(ctx, "proj", "exp1"); err != nil { + t.Fatal(err) + } results, err := client.Experiments.Results(ctx, "proj", "exp1", api.ExperimentResultsOptions{Platform: "ios", Country: "US", ExposureStatus: "exposed", Currency: "EUR"}) if err != nil { t.Fatal(err) @@ -49,6 +52,7 @@ func TestExperimentsReadRoutesAndFilters(t *testing.T) { want := []string{ "GET /projects/proj/experiments?limit=5&starting_after=exp0&status=running", "GET /projects/proj/experiments/exp1", + "GET /projects/proj/experiments/exp1?expand=offering.paywall", "GET /projects/proj/experiments/exp1/results?country=US¤cy=EUR&exposure_status=exposed&platform=ios", } if fmt.Sprint(requests) != fmt.Sprint(want) { diff --git a/internal/cli/experiment_format.go b/internal/cli/experiment_format.go index 24a2ea91..95390cb4 100644 --- a/internal/cli/experiment_format.go +++ b/internal/cli/experiment_format.go @@ -130,6 +130,13 @@ func renderExperimentShow(rt *Runtime, experiment *api.Experiment) error { return nil } +func experimentOfferingID(offering *api.ExperimentOffering) string { + if offering == nil { + return "" + } + return offering.ID +} + func experimentOfferingLabel(offering *api.ExperimentOffering) string { label := offering.ID if offering.DisplayName != "" { diff --git a/internal/cli/experiments.go b/internal/cli/experiments.go index 3fba5887..437569d9 100644 --- a/internal/cli/experiments.go +++ b/internal/cli/experiments.go @@ -28,7 +28,8 @@ func newExperimentsListCmd() *cobra.Command { cmd := &cobra.Command{ Use: "list", Short: "List experiments", - Example: " rc experiments list --status running\n rc experiments list --json", + Long: "Returns one page of experiments. Use --limit to set the page size and --cursor for the next page. JSON includes next_page.", + Example: " rc experiments list --status running\n rc experiments list --cursor exp123 --json", RunE: func(cmd *cobra.Command, _ []string) error { rt := RuntimeFrom(cmd.Context()) projectID, err := requireProject(rt) @@ -45,19 +46,32 @@ func newExperimentsListCmd() *cobra.Command { if err != nil { return err } + hasC, hasD := false, false + for _, experiment := range page.Items { + hasC = hasC || experiment.OfferingC != nil + hasD = hasD || experiment.OfferingD != nil + } + columns := []string{"ID", "NAME", "STATUS", "CONTROL", "TREATMENT"} + if hasC { + columns = append(columns, "TREATMENT C") + } + if hasD { + columns = append(columns, "TREATMENT D") + } rows := make([][]string, 0, len(page.Items)) for _, experiment := range page.Items { - control, treatment := "", "" - if experiment.OfferingA != nil { - control = experiment.OfferingA.ID + row := []string{experiment.ID, experiment.DisplayName, experiment.Status, + experimentOfferingID(experiment.OfferingA), experimentOfferingID(experiment.OfferingB)} + if hasC { + row = append(row, experimentOfferingID(experiment.OfferingC)) } - if experiment.OfferingB != nil { - treatment = experiment.OfferingB.ID + if hasD { + row = append(row, experimentOfferingID(experiment.OfferingD)) } - rows = append(rows, []string{experiment.ID, experiment.DisplayName, experiment.Status, control, treatment}) + rows = append(rows, row) } if err := rt.Out.RenderTable(output.Table{ - Columns: []string{"ID", "NAME", "STATUS", "CONTROL", "TREATMENT"}, + Columns: columns, Rows: rows, Raw: page, }); err != nil { return err @@ -73,15 +87,21 @@ func newExperimentsListCmd() *cobra.Command { } func experimentPickerItems(cmd *cobra.Command, client *api.Client, projectID string) ([]PickerItem, error) { - page, err := client.Experiments.List(cmd.Context(), projectID, api.ListExperimentsOptions{}) - if err != nil { - return nil, err - } - items := make([]PickerItem, len(page.Items)) - for i, experiment := range page.Items { - items[i] = PickerItem{ID: experiment.ID, Label: fmt.Sprintf("%s (%s)", experiment.DisplayName, experiment.Status)} + items := []PickerItem{} + cursor := "" + for { + page, err := client.Experiments.List(cmd.Context(), projectID, api.ListExperimentsOptions{Limit: 100, StartingAfter: cursor}) + if err != nil { + return nil, err + } + for _, experiment := range page.Items { + items = append(items, PickerItem{ID: experiment.ID, Label: fmt.Sprintf("%s (%s)", experiment.DisplayName, experiment.Status)}) + } + cursor = page.NextCursor() + if cursor == "" { + return items, nil + } } - return items, nil } func newExperimentsShowCmd() *cobra.Command { @@ -106,7 +126,7 @@ func newExperimentsShowCmd() *cobra.Command { if err != nil { return err } - experiment, err := client.Experiments.Get(cmd.Context(), projectID, id) + experiment, err := client.Experiments.GetWithPaywalls(cmd.Context(), projectID, id) if err != nil { return err } @@ -158,6 +178,16 @@ func newExperimentsResultsCmd() *cobra.Command { if rt.Globals.JSON { return rt.Out.Render(results) } + rt.Out.Title("Experiment results") + if opts.Platform != "" { + rt.Out.Field("Platform", opts.Platform) + } + if opts.Country != "" { + rt.Out.Field("Country", opts.Country) + } + if opts.ExposureStatus != "" { + rt.Out.Field("Exposure", opts.ExposureStatus) + } return renderExperimentResults(rt, results) }, } @@ -212,7 +242,6 @@ func experimentExposureStatus(value string) (string, error) { } func renderExperimentResults(rt *Runtime, results *api.ExperimentResults) error { - rt.Out.Title("Experiment results") rt.Out.Field("Currency", results.Currency) shownSections := 0 for _, section := range results.Sections { @@ -313,7 +342,7 @@ func chanceToWinLabel(stat api.ExperimentResultVariantStatistic) string { func liftIntervalLabel(stat api.ExperimentResultVariantStatistic) string { if stat.LiftCredibleIntervalLower != nil && stat.LiftCredibleIntervalUpper != nil { - return fmt.Sprintf("%+.1f%% to %+.1f%%", *stat.LiftCredibleIntervalLower*100, *stat.LiftCredibleIntervalUpper*100) + return fmt.Sprintf("%+.1f%% to %+.1f%%", *stat.LiftCredibleIntervalLower, *stat.LiftCredibleIntervalUpper) } if stat.LiftCredibleIntervalStatus == "insufficient_data" { return "Need more data" diff --git a/internal/cli/experiments_picker_test.go b/internal/cli/experiments_picker_test.go new file mode 100644 index 00000000..0da4f789 --- /dev/null +++ b/internal/cli/experiments_picker_test.go @@ -0,0 +1,39 @@ +package cli + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/revenuecat/cli/internal/api" + "github.com/spf13/cobra" +) + +func TestExperimentPickerIncludesNextPage(t *testing.T) { + requests := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.URL.Query().Get("limit") != "100" { + t.Errorf("limit=%s", r.URL.Query().Get("limit")) + } + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("starting_after") == "" { + fmt.Fprint(w, `{"items":[{"id":"exp1","display_name":"First","status":"draft"}],"next_page":"/projects/proj/experiments?starting_after=exp1"}`) + } else { + if r.URL.Query().Get("starting_after") != "exp1" { + t.Errorf("cursor=%s", r.URL.RawQuery) + } + fmt.Fprint(w, `{"items":[{"id":"exp2","display_name":"Second","status":"running"}],"next_page":null}`) + } + })) + defer srv.Close() + client := api.NewClient(api.Options{BaseURL: srv.URL, APIKey: "sk_test"}) + cmd := &cobra.Command{} + cmd.SetContext(context.Background()) + items, err := experimentPickerItems(cmd, client, "proj") + if err != nil || requests != 2 || len(items) != 2 || items[1].ID != "exp2" { + t.Fatalf("items=%v requests=%d err=%v", items, requests, err) + } +} diff --git a/internal/cli/experiments_test.go b/internal/cli/experiments_test.go index fcfaf633..0001258a 100644 --- a/internal/cli/experiments_test.go +++ b/internal/cli/experiments_test.go @@ -132,3 +132,93 @@ func TestExperimentsDiscoverable(t *testing.T) { t.Fatalf("results command not discoverable as a default command: %s", out) } } + +func TestExperimentResultsPreservesPercentageUnitsAndShowsFilters(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"currency":"USD","sections":[],"statistics":[{"metric_name":"trial_conversion_rate","variants":[{"name":"Treatment","chance_to_win":0.98,"lift_credible_interval_lower":-5,"lift_credible_interval_upper":22}]}]}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + out, stderr, err := runAgentCmd(t, "experiments", "results", "exp1", "--platform", "ios", "--country", "us", "--exposure-status", "exposed", "--project-id", "proj", "--api-key", "sk_test", "--no-input") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"98.0%", "-5.0% to +22.0%", "iOS", "US", "exposed"} { + if !strings.Contains(out+stderr, want) { + t.Fatalf("missing %q: stdout=%s stderr=%s", want, out, stderr) + } + } +} + +func TestExperimentsListIncludesFourVariantsAndPagination(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + if q.Get("limit") != "5" || q.Get("starting_after") != "exp0" || q.Get("status") != "running" { + t.Errorf("unexpected query: %s", r.URL.RawQuery) + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"items":[{"id":"exp1","display_name":"Four variants","status":"running","offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"},"offering_c":{"id":"ofrng_c"},"offering_d":{"id":"ofrng_d"}}],"next_page":"/projects/proj/experiments?starting_after=exp1"}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + args := []string{"experiments", "list", "--status", "running", "--limit", "5", "--cursor", "exp0", "--project-id", "proj", "--api-key", "sk_test", "--no-input"} + out, stderr, err := runAgentCmd(t, args...) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"ofrng_a", "ofrng_b", "ofrng_c", "ofrng_d", "--cursor exp1"} { + if !strings.Contains(out+stderr, want) { + t.Fatalf("missing %q: stdout=%s stderr=%s", want, out, stderr) + } + } + out, stderr, err = runAgentCmd(t, append(args, "--json")...) + if err != nil || stderr != "" || !strings.Contains(out, `"next_page"`) || !strings.Contains(out, "ofrng_d") { + t.Fatalf("pagination or variants missing in JSON: err=%v stdout=%s stderr=%s", err, out, stderr) + } +} + +func TestExperimentShowIncludesPaywallsAndTargeting(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("expand") != "offering.paywall" { + t.Errorf("missing paywall expansion: %s", r.URL.RawQuery) + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Four variants","status":"draft","offering_a":{"id":"ofrng_a","paywall_id":"pw_a"},"offering_b":{"id":"ofrng_b"},"offering_c":{"id":"ofrng_c","paywall_id":"pw_c"},"offering_d":{"id":"ofrng_d"},"targeting_conditions":[{"field":"platform","operator":"in","value":["ios"]},{"field":"country","operator":"in","value":["US"]}]}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + for _, jsonOutput := range []bool{false, true} { + args := []string{"experiments", "show", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input"} + if jsonOutput { + args = append(args, "--json") + } + out, _, err := runAgentCmd(t, args...) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"pw_a", "pw_c", "ofrng_c", "ofrng_d", "platform", "ios", "country", "US"} { + if !strings.Contains(out, want) { + t.Fatalf("missing %q (json=%v): %s", want, jsonOutput, out) + } + } + } +} + +func TestExperimentResultsIncludesFourVariants(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"currency":"USD","sections":[{"section":"Conversion","metrics":[{"name":"trial_conversion_rate","unit":"%"}],"segments":[{"is_total":true}],"values":[{"metric":0,"segment":0,"variant":"Control","value":10},{"metric":0,"segment":0,"variant":"Treatment","value":11,"change":10},{"metric":0,"segment":0,"variant":"TreatmentC","value":12,"change":20},{"metric":0,"segment":0,"variant":"TreatmentD","value":13,"change":30}]}],"statistics":[]}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + out, _, err := runAgentCmd(t, "experiments", "results", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"TREATMENT C", "TREATMENT D", "+10.00%", "+20.00%", "+30.00%"} { + if !strings.Contains(out, want) { + t.Fatalf("missing %q: %s", want, out) + } + } +} diff --git a/internal/cli/snapshot_test.go b/internal/cli/snapshot_test.go index f9e8394f..2b6ebcfc 100644 --- a/internal/cli/snapshot_test.go +++ b/internal/cli/snapshot_test.go @@ -13,6 +13,7 @@ package cli_test // internal/output/brand.go and are reviewed there. import ( + "context" "fmt" "io" "net/http" @@ -21,6 +22,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/revenuecat/cli/internal/cli" ) func snapshotServer(t *testing.T) *httptest.Server { @@ -28,10 +31,12 @@ func snapshotServer(t *testing.T) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") switch { + case strings.HasSuffix(r.URL.Path, "/experiments"): + io.WriteString(w, `{"items":[{"id":"exp_snap","display_name":"Four variants","status":"draft","offering_a":{"id":"ofrng_a"},"offering_b":{"id":"ofrng_b"},"offering_c":{"id":"ofrng_c"},"offering_d":{"id":"ofrng_d"}}],"next_page":null}`) case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap"): io.WriteString(w, `{"object":"experiment","id":"exp_snap","display_name":"Price — paywall","status":"paused","created_at":1784297950368,"updated_at":1784297950368,"total_running_time_seconds":1989824,"enrollment_percentage":50,"offering_a":{"id":"ofrng_control","display_name":"Control"},"offering_b":{"id":"ofrng_treatment","display_name":"Treatment"},"targeting_conditions":[{"field":"platform","operator":"in","value":["ios"]}],"placements":{"placement_offerings":[{"placement_identifier":"onboarding","offering_a":{"id":"ofrng_control"},"offering_b":{"id":"ofrng_treatment"}}]},"primary_metric":"initial_conversion_rate","secondary_metrics":["realized_ltv_per_customer","trial_conversion_rate"],"experiment_duration_settings":{"chance_to_win_percentage":95}}`) case strings.HasSuffix(r.URL.Path, "/experiments/exp_snap/results"): - io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[{"object":"experiment_results_section","section":"Conversion","metrics":[{"name":"trial_conversion_rate","unit":"%"}],"segments":[{"object":"experiment_results_segment","id":"total","display_name":"Total","is_total":true}],"values":[{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Control","value":12.5,"change":null,"credible_interval":null,"lift_credible_interval":null},{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Treatment","value":15.0,"change":20,"credible_interval":null,"lift_credible_interval":null}]}],"statistics":[{"object":"experiment_results_statistic","metric_name":"trial_conversion_rate","variants":[{"object":"experiment_results_variant_statistic","name":"Control","chance_to_win":null,"chance_to_win_status":"unavailable","credible_interval_lower":0.1,"credible_interval_upper":0.15,"credible_interval_status":"ok","lift_credible_interval_lower":null,"lift_credible_interval_upper":null,"lift_credible_interval_status":"unavailable"},{"object":"experiment_results_variant_statistic","name":"Treatment","chance_to_win":0.98,"chance_to_win_status":"ok","credible_interval_lower":0.11,"credible_interval_upper":0.18,"credible_interval_status":"ok","lift_credible_interval_lower":-0.05,"lift_credible_interval_upper":0.22,"lift_credible_interval_status":"ok"}]}],"predicted_ltv":null}`) + io.WriteString(w, `{"object":"experiment_results","currency":"USD","sections":[{"object":"experiment_results_section","section":"Conversion","metrics":[{"name":"trial_conversion_rate","unit":"%"}],"segments":[{"object":"experiment_results_segment","id":"total","display_name":"Total","is_total":true}],"values":[{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Control","value":12.5,"change":null,"credible_interval":null,"lift_credible_interval":null},{"object":"experiment_results_value","metric":0,"segment":0,"variant":"Treatment","value":15.0,"change":20,"credible_interval":null,"lift_credible_interval":null}]}],"statistics":[{"object":"experiment_results_statistic","metric_name":"trial_conversion_rate","variants":[{"object":"experiment_results_variant_statistic","name":"Control","chance_to_win":null,"chance_to_win_status":"unavailable","credible_interval_lower":0.1,"credible_interval_upper":0.15,"credible_interval_status":"ok","lift_credible_interval_lower":null,"lift_credible_interval_upper":null,"lift_credible_interval_status":"unavailable"},{"object":"experiment_results_variant_statistic","name":"Treatment","chance_to_win":0.98,"chance_to_win_status":"ok","credible_interval_lower":0.11,"credible_interval_upper":0.18,"credible_interval_status":"ok","lift_credible_interval_lower":-5,"lift_credible_interval_upper":22,"lift_credible_interval_status":"ok"}]}],"predicted_ltv":null}`) case strings.HasSuffix(r.URL.Path, "/offerings/ofrng_snap"): io.WriteString(w, `{"object":"offering","id":"ofrng_snap","lookup_key":"default","display_name":"Default","is_current":true,"created_at":1784297950368,"project_id":"proj_snap"}`) case strings.HasSuffix(r.URL.Path, "/apps/app_snap1"): @@ -62,6 +67,7 @@ func TestOutputSnapshots(t *testing.T) { {"version", []string{"version"}}, {"auth-status-logged-out", []string{"auth", "status", "--no-input"}}, {"offerings-show", []string{"offerings", "show", "ofrng_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, + {"experiments-list", []string{"experiments", "list", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"experiments-show", []string{"experiments", "show", "exp_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"experiments-results", []string{"experiments", "results", "exp_snap", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, {"apps-list", []string{"apps", "list", "--no-input", "--project-id", "proj_snap", "--api-key", "sk_snap"}}, @@ -72,15 +78,15 @@ func TestOutputSnapshots(t *testing.T) { for _, sc := range scenarios { t.Run(sc.name, func(t *testing.T) { - stdout, stderr, err := runAgentCmd(t, sc.args...) + t.Setenv("RC_CONFIG_DIR", t.TempDir()) var b strings.Builder fmt.Fprintf(&b, "$ rc %s\n", strings.Join(sc.args, " ")) - if stderr != "" { - b.WriteString(stderr) - } - if stdout != "" { - b.WriteString(stdout) - } + root := cli.NewRootCmd("test") + // A shared writer preserves terminal ordering across stdout and stderr. + root.SetOut(&b) + root.SetErr(&b) + root.SetArgs(sc.args) + err := root.ExecuteContext(context.Background()) if err != nil { fmt.Fprintf(&b, "(exit: %v)\n", err) } diff --git a/internal/cli/testdata/snapshots/apps-list-all-projects.golden b/internal/cli/testdata/snapshots/apps-list-all-projects.golden index 4125fb27..afb3071b 100644 --- a/internal/cli/testdata/snapshots/apps-list-all-projects.golden +++ b/internal/cli/testdata/snapshots/apps-list-all-projects.golden @@ -1,6 +1,6 @@ $ rc apps list --all-projects --bundle-id com.example.moodly --no-input --api-key sk_snap -! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. - Fix it: rc setup apple app_snap1 --project-id proj_snap (interactive Apple sign-in with 2FA) PROJECT ID NAME TYPE CREATED CREDENTIALS proj_snap app_snap1 Moodly (App Store) app_store 2026-07-16 partial — run: rc setup apple proj_snap2 app_snap3 Moodly Staging (App Store) app_store 2026-07-16 ready +! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. + Fix it: rc setup apple app_snap1 --project-id proj_snap (interactive Apple sign-in with 2FA) diff --git a/internal/cli/testdata/snapshots/apps-list.golden b/internal/cli/testdata/snapshots/apps-list.golden index ba8db20b..c7e8af6e 100644 --- a/internal/cli/testdata/snapshots/apps-list.golden +++ b/internal/cli/testdata/snapshots/apps-list.golden @@ -1,6 +1,6 @@ $ rc apps list --no-input --project-id proj_snap --api-key sk_snap -! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. - Fix it: rc setup apple app_snap1 (interactive Apple sign-in with 2FA) ID NAME TYPE CREATED CREDENTIALS app_snap1 Moodly (App Store) app_store 2026-07-16 partial — run: rc setup apple app_snap2 Test Store test_store 2026-07-16 — +! app_snap1 is missing Apple credentials — App Store purchases can't be validated until they're set. + Fix it: rc setup apple app_snap1 (interactive Apple sign-in with 2FA) diff --git a/internal/cli/testdata/snapshots/experiments-list.golden b/internal/cli/testdata/snapshots/experiments-list.golden new file mode 100644 index 00000000..62511d45 --- /dev/null +++ b/internal/cli/testdata/snapshots/experiments-list.golden @@ -0,0 +1,3 @@ +$ rc experiments list --no-input --project-id proj_snap --api-key sk_snap +ID NAME STATUS CONTROL TREATMENT TREATMENT C TREATMENT D +exp_snap Four variants draft ofrng_a ofrng_b ofrng_c ofrng_d diff --git a/internal/cli/testdata/snapshots/experiments-results.golden b/internal/cli/testdata/snapshots/experiments-results.golden index f3c1052f..90956aaf 100644 --- a/internal/cli/testdata/snapshots/experiments-results.golden +++ b/internal/cli/testdata/snapshots/experiments-results.golden @@ -4,9 +4,9 @@ $ rc experiments results exp_snap --no-input --project-id proj_snap --api-key sk Currency USD ▍ Conversion - -▍ Decision signals METRIC CONTROL TREATMENT CHANGE UNIT trial_conversion_rate 12.5 15 +20.00% % + +▍ Decision signals METRIC VARIANT CHANCE TO WIN 95% LIFT INTERVAL trial_conversion_rate Treatment 98.0% -5.0% to +22.0% diff --git a/internal/cli/testdata/snapshots/experiments-show.golden b/internal/cli/testdata/snapshots/experiments-show.golden index b4492560..e7f18c55 100644 --- a/internal/cli/testdata/snapshots/experiments-show.golden +++ b/internal/cli/testdata/snapshots/experiments-show.golden @@ -1,5 +1,4 @@ $ rc experiments show exp_snap --no-input --project-id proj_snap --api-key sk_snap - Use --json for the complete API response. ▍ Price — paywall exp_snap · paused @@ -21,3 +20,4 @@ Setup Timeline Running time: 23d 0h 43m + Use --json for the complete API response. From 9ca48667054fbac893207338167a31504bc18a99 Mon Sep 17 00:00:00 2001 From: Josh Holtz Date: Wed, 7 Oct 2026 08:16:11 -0500 Subject: [PATCH 7/7] fix(experiments): fall back when paywall expansion is forbidden --- internal/cli/experiments.go | 9 +++++ internal/cli/experiments_test.go | 63 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/internal/cli/experiments.go b/internal/cli/experiments.go index 437569d9..b0ca8a1f 100644 --- a/internal/cli/experiments.go +++ b/internal/cli/experiments.go @@ -2,7 +2,9 @@ package cli import ( "encoding/json" + "errors" "fmt" + "net/http" "strconv" "strings" @@ -127,6 +129,13 @@ func newExperimentsShowCmd() *cobra.Command { return err } experiment, err := client.Experiments.GetWithPaywalls(cmd.Context(), projectID, id) + var apiErr *api.APIError + if errors.As(err, &apiErr) && apiErr.Status == http.StatusForbidden { + experiment, err = client.Experiments.Get(cmd.Context(), projectID, id) + if err == nil { + rt.Out.AlwaysWarn("Paywall details are unavailable; they require project_configuration:offerings:read permission.") + } + } if err != nil { return err } diff --git a/internal/cli/experiments_test.go b/internal/cli/experiments_test.go index 0001258a..235b9699 100644 --- a/internal/cli/experiments_test.go +++ b/internal/cli/experiments_test.go @@ -222,3 +222,66 @@ func TestExperimentResultsIncludesFourVariants(t *testing.T) { } } } + +func TestExperimentShowFallsBackOnlyOnExpandedForbidden(t *testing.T) { + for _, tc := range []struct { + name string + expandedStatus, plainStatus, requests int + jsonMode, success bool + }{ + {"human fallback", 403, 200, 2, false, true}, + {"JSON fallback", 403, 200, 2, true, true}, + {"fallback forbidden", 403, 403, 2, true, false}, + {"not found", 404, 200, 1, true, false}, + {"unauthorized", 401, 200, 1, true, false}, + {"invalid request", 422, 200, 1, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + requests := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + status := tc.expandedStatus + if requests == 1 { + if r.URL.Query().Get("expand") != "offering.paywall" { + t.Errorf("missing expansion: %s", r.URL.RawQuery) + } + } else { + if r.URL.Query().Has("expand") { + t.Errorf("fallback still expanded: %s", r.URL.RawQuery) + } + status = tc.plainStatus + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if status != http.StatusOK { + _, _ = io.WriteString(w, `{"type":"forbidden","message":"Denied"}`) + return + } + _, _ = io.WriteString(w, `{"id":"exp1","display_name":"Test","status":"draft","offering_a":{"id":"ofrng_a"}}`) + })) + t.Cleanup(srv.Close) + t.Setenv("RC_BASE_URL", srv.URL) + args := []string{"experiments", "show", "exp1", "--project-id", "proj", "--api-key", "sk_test", "--no-input"} + if tc.jsonMode { + args = append(args, "--json") + } + out, stderr, err := runAgentCmd(t, args...) + if (err == nil) != tc.success || requests != tc.requests { + t.Fatalf("err=%v requests=%d want=%d", err, requests, tc.requests) + } + if tc.success { + if !strings.Contains(out, "exp1") || !strings.Contains(stderr, "project_configuration:offerings:read") { + t.Fatalf("missing experiment or warning: stdout=%s stderr=%s", out, stderr) + } + if tc.jsonMode { + var envelope map[string]any + if err := json.Unmarshal([]byte(out), &envelope); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + } + } else if strings.Contains(stderr, "Paywall details are unavailable") { + t.Fatalf("warned despite failed fallback: %s", stderr) + } + }) + } +}