From 3ea9dc0d2f02aabcf39d41edf3a9d0d9a95182bf Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" Date: Tue, 11 Aug 2026 15:16:05 +0800 Subject: [PATCH 01/48] feat: add safe PPE request routing Co-authored-by: Codex --- internal/common/client.go | 178 +++++++++++++++++--- internal/common/client_test.go | 286 +++++++++++++++++++++++++++++++++ internal/config/config.go | 21 +++ internal/config/config_test.go | 45 ++++++ 4 files changed, 509 insertions(+), 21 deletions(-) create mode 100644 internal/common/client_test.go diff --git a/internal/common/client.go b/internal/common/client.go index a10c075..e4a46b1 100644 --- a/internal/common/client.go +++ b/internal/common/client.go @@ -15,9 +15,17 @@ import ( "strings" "time" + "github.com/Pippit-dev/pippit-cli/internal/config" "github.com/bytedance/sonic" ) +const ( + ppeUseHeader = "x-use-ppe" + ppeEnvHeader = "x-tt-env" + ppeScheduleVDCHeader = "x-schedule-vdc" + defaultPPEVDC = "sinfonlinea" +) + type Client interface { SendRequest(ctx context.Context, path string, body any, out any) error SendRequestWithHeaders(ctx context.Context, path string, body any, headers map[string]string, out any) error @@ -40,17 +48,32 @@ type httpClient struct { httpClient *http.Client headers http.Header authorizer RequestAuthorizer + ppeEnv func() string } func NewHTTPClient(baseURL string, timeout time.Duration, authorizer RequestAuthorizer) Client { - return &httpClient{ - baseURL: strings.TrimRight(baseURL, "/"), - httpClient: &http.Client{ - Timeout: timeout, - }, + return newHTTPClient(baseURL, timeout, authorizer, nil) +} + +// NewHTTPClientWithPPEEnv creates a client whose PPE lane is resolved for each +// request. The provider allows Cobra's global --ppe-env flag to override the +// environment after command construction but before the request is sent. +func NewHTTPClientWithPPEEnv(baseURL string, timeout time.Duration, authorizer RequestAuthorizer, ppeEnv func() string) Client { + return newHTTPClient(baseURL, timeout, authorizer, ppeEnv) +} + +func newHTTPClient(baseURL string, timeout time.Duration, authorizer RequestAuthorizer, ppeEnv func() string) Client { + client := &httpClient{ + baseURL: strings.TrimRight(baseURL, "/"), headers: make(http.Header), authorizer: authorizer, + ppeEnv: ppeEnv, + } + client.httpClient = &http.Client{ + Timeout: timeout, + CheckRedirect: client.checkRedirect, } + return client } func (c *httpClient) SendRequest(ctx context.Context, path string, body any, out any) error { @@ -85,15 +108,10 @@ func (c *httpClient) SendRequestWithHeaders(ctx context.Context, path string, bo req.Header.Set("Content-Type", "application/json") } - if c.authorizer == nil { - return fmt.Errorf("授权请求缺少认证器") - } - if err := c.authorizer.Inject(ctx, req); err != nil { - return fmt.Errorf("写入认证请求头失败: %w", err) + if err := c.prepareRequest(ctx, req, headers); err != nil { + return err } - c.injectHeaders(req, headers) - // If out is **http.Response, return the raw response for streaming (e.g. file download). if out != nil { if rv := reflect.ValueOf(out); rv.Kind() == reflect.Ptr && rv.Elem().Kind() == reflect.Ptr { @@ -143,17 +161,11 @@ func (c *httpClient) SendMultipartRequest(ctx context.Context, path string, fiel req.Header.Set("Content-Type", writer.FormDataContentType()) req.Header.Set("Accept", "application/json") - if c.authorizer == nil { + if err := c.prepareRequest(ctx, req, nil); err != nil { _ = pr.Close() _ = pw.Close() - return fmt.Errorf("授权请求缺少认证器") - } - if err := c.authorizer.Inject(ctx, req); err != nil { - _ = pr.Close() - _ = pw.Close() - return fmt.Errorf("写入认证请求头失败: %w", err) + return err } - c.injectHeaders(req, nil) go func() { err := writeMultipartBody(writer, fields, file) @@ -212,6 +224,123 @@ func (c *httpClient) injectHeaders(req *http.Request, headers map[string]string) } } +func (c *httpClient) prepareRequest(ctx context.Context, req *http.Request, headers map[string]string) error { + trusted, err := c.isBaseURLOrigin(req.URL) + if err != nil { + return err + } + + var ppeEnv string + if trusted { + if c.ppeEnv != nil { + ppeEnv, err = config.NormalizePPEEnv(c.ppeEnv()) + if err != nil { + return err + } + } + } + + c.injectHeaders(req, headers) + // Authentication and PPE routing are protected headers. Neither the + // client's generic headers nor a caller-provided map may set or override + // them; rebuild them below only for the configured API origin. + req.Header.Del("Authorization") + req.Header.Del(ppeUseHeader) + req.Header.Del(ppeEnvHeader) + req.Header.Del(ppeScheduleVDCHeader) + if !trusted { + // Absolute third-party URLs are used for result downloads. The protected + // headers remain empty outside the API origin. + return nil + } + if c.authorizer == nil { + return fmt.Errorf("授权请求缺少认证器") + } + if err := c.authorizer.Inject(ctx, req); err != nil { + return fmt.Errorf("写入认证请求头失败: %w", err) + } + if ppeEnv != "" { + req.Header.Set(ppeUseHeader, "1") + req.Header.Set(ppeEnvHeader, ppeEnv) + req.Header.Set(ppeScheduleVDCHeader, defaultPPEVDC) + } + return nil +} + +func (c *httpClient) isBaseURLOrigin(target *url.URL) (bool, error) { + if c.baseURL == "" { + return false, nil + } + base, err := url.Parse(c.baseURL) + if err != nil || base.Scheme == "" || base.Host == "" { + return false, fmt.Errorf("解析 base URL %q 失败", c.baseURL) + } + return sameOrigin(base, target), nil +} + +func (c *httpClient) checkRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("停止重定向：已达到 10 次上限") + } + trusted, err := c.isBaseURLOrigin(req.URL) + if err != nil { + return err + } + initialTrusted, err := c.isBaseURLOrigin(via[0].URL) + if err != nil { + return err + } + if initialTrusted && !trusted { + // API requests may contain unreleased canvas data or upload metadata. + // Stripping credentials is insufficient because Go can preserve the + // method and body for 307/308 redirects. API calls have no valid reason + // to leave the configured origin, so reject the redirect entirely. + return fmt.Errorf("拒绝 Pippit API 跨域重定向到 %s", req.URL.Redacted()) + } + if trusted { + // net/http may rebuild redirect headers from the original request. Once a + // chain has crossed an untrusted origin, never restore protected headers + // even if a later hop points back at the API origin. + for _, previous := range via { + previousTrusted, err := c.isBaseURLOrigin(previous.URL) + if err != nil { + return err + } + if !previousTrusted { + trusted = false + break + } + } + } + if !trusted { + req.Header.Del("Authorization") + req.Header.Del(ppeUseHeader) + req.Header.Del(ppeEnvHeader) + req.Header.Del(ppeScheduleVDCHeader) + } + return nil +} + +func sameOrigin(a, b *url.URL) bool { + return strings.EqualFold(a.Scheme, b.Scheme) && + strings.EqualFold(a.Hostname(), b.Hostname()) && + effectivePort(a) == effectivePort(b) +} + +func effectivePort(u *url.URL) string { + if port := u.Port(); port != "" { + return port + } + switch strings.ToLower(u.Scheme) { + case "http": + return "80" + case "https": + return "443" + default: + return "" + } +} + func (c *httpClient) do(req *http.Request, out any) error { resp, err := c.httpClient.Do(req) if err != nil { @@ -240,7 +369,14 @@ func (c *httpClient) do(req *http.Request, out any) error { } func (c *httpClient) resolveURL(path string, query map[string]string) (string, error) { - if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + parsed, err := url.Parse(path) + if err != nil { + return "", fmt.Errorf("解析 URL 失败: %w", err) + } + if parsed.IsAbs() { + if parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") { + return "", fmt.Errorf("仅支持 http 或 https URL: %q", path) + } return appendQuery(path, query) } if c.baseURL == "" { diff --git a/internal/common/client_test.go b/internal/common/client_test.go new file mode 100644 index 0000000..a9dcb13 --- /dev/null +++ b/internal/common/client_test.go @@ -0,0 +1,286 @@ +package common + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +type observedHeaders struct { + authorization string + usePPE string + ppeEnv string + scheduleVDC string +} + +func TestHTTPClientScopesCredentialsAndPPEToBaseOrigin(t *testing.T) { + apiHeaders := make(chan observedHeaders, 1) + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + apiHeaders <- captureRoutingHeaders(r) + _, _ = w.Write([]byte(`{}`)) + })) + defer api.Close() + + thirdPartyHeaders := make(chan observedHeaders, 1) + thirdParty := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + thirdPartyHeaders <- captureRoutingHeaders(r) + _, _ = w.Write([]byte(`{}`)) + })) + defer thirdParty.Close() + + client := NewHTTPClientWithPPEEnv( + api.URL, + time.Second, + NewAccessKeyAuthorizer("secret-ak"), + func() string { return "ppe_cli_canvas_ak" }, + ) + if err := client.SendRequest(context.Background(), "/api/test", nil, nil); err != nil { + t.Fatalf("same-origin SendRequest() error = %v", err) + } + gotAPI := <-apiHeaders + if gotAPI.authorization != "Bearer secret-ak" { + t.Fatalf("same-origin Authorization = %q", gotAPI.authorization) + } + if gotAPI.usePPE != "1" || gotAPI.ppeEnv != "ppe_cli_canvas_ak" || gotAPI.scheduleVDC != defaultPPEVDC { + t.Fatalf("same-origin PPE headers = %#v", gotAPI) + } + + thirdPartyURL := thirdParty.URL + "/asset.mp4" + err := client.SendRequestWithHeaders(context.Background(), thirdPartyURL, nil, map[string]string{ + "Authorization": "Bearer caller-supplied", + "x-use-ppe": "1", + "x-tt-env": "ppe_leak", + "x-schedule-vdc": "leak-vdc", + }, nil) + if err != nil { + t.Fatalf("third-party SendRequest() error = %v", err) + } + gotThirdParty := <-thirdPartyHeaders + if gotThirdParty != (observedHeaders{}) { + t.Fatalf("third-party sensitive headers leaked: %#v", gotThirdParty) + } +} + +func TestHTTPClientTreatsSameOriginAbsoluteURLAsAPI(t *testing.T) { + headers := make(chan observedHeaders, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + headers <- captureRoutingHeaders(r) + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + + client := NewHTTPClientWithPPEEnv( + server.URL+"/base-path", + time.Second, + NewAccessKeyAuthorizer("same-origin-ak"), + func() string { return "ppe_absolute" }, + ) + if err := client.SendRequest(context.Background(), server.URL+"/api/absolute", nil, nil); err != nil { + t.Fatalf("SendRequest() error = %v", err) + } + got := <-headers + if got.authorization != "Bearer same-origin-ak" || got.usePPE != "1" || got.ppeEnv != "ppe_absolute" || got.scheduleVDC != defaultPPEVDC { + t.Fatalf("same-origin absolute headers = %#v", got) + } +} + +func TestHTTPClientRejectsAPICrossOriginRedirectBeforeSendingBody(t *testing.T) { + var thirdPartyRequests atomic.Int32 + thirdParty := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + thirdPartyRequests.Add(1) + _, _ = w.Write([]byte(`{}`)) + })) + defer thirdParty.Close() + + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, thirdParty.URL+"/redirected-asset", http.StatusTemporaryRedirect) + })) + defer api.Close() + + client := NewHTTPClientWithPPEEnv( + api.URL, + time.Second, + NewAccessKeyAuthorizer("redirect-ak"), + func() string { return "ppe_redirect" }, + ) + err := client.SendRequest(context.Background(), "/api/redirect", map[string]string{"canvas": "private"}, nil) + if err == nil || !strings.Contains(err.Error(), "拒绝 Pippit API 跨域重定向") { + t.Fatalf("SendRequest() error = %v, want cross-origin redirect rejection", err) + } + if got := thirdPartyRequests.Load(); got != 0 { + t.Fatalf("third party received %d API redirect requests, want 0", got) + } +} + +func TestHTTPClientDoesNotRestoreProtectedHeadersAfterCrossOriginRedirect(t *testing.T) { + finalHeaders := make(chan observedHeaders, 1) + var apiURL string + thirdParty := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := captureRoutingHeaders(r); got != (observedHeaders{}) { + t.Errorf("cross-origin hop leaked protected headers: %#v", got) + } + http.Redirect(w, r, apiURL+"/final", http.StatusFound) + })) + defer thirdParty.Close() + + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + finalHeaders <- captureRoutingHeaders(r) + _, _ = w.Write([]byte(`{}`)) + })) + defer api.Close() + apiURL = api.URL + + client := NewHTTPClientWithPPEEnv( + api.URL, + time.Second, + NewAccessKeyAuthorizer("redirect-bounce-ak"), + func() string { return "ppe_redirect_bounce" }, + ) + if err := client.SendRequest(context.Background(), thirdParty.URL+"/bounce", nil, nil); err != nil { + t.Fatalf("SendRequest() error = %v", err) + } + if got := <-finalHeaders; got != (observedHeaders{}) { + t.Fatalf("trusted return hop restored protected headers: %#v", got) + } +} + +func TestHTTPClientRejectsInvalidPPEEnvBeforeNetwork(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + + client := NewHTTPClientWithPPEEnv( + server.URL, + time.Second, + NewAccessKeyAuthorizer("secret-ak"), + func() string { return "production" }, + ) + err := client.SendRequest(context.Background(), "/api/test", nil, nil) + if err == nil || !strings.Contains(err.Error(), "PPE 环境") { + t.Fatalf("SendRequest() error = %v, want invalid PPE error", err) + } + if got := requests.Load(); got != 0 { + t.Fatalf("server received %d requests, want 0", got) + } +} + +func TestHTTPClientProductionRequestOmitsPPEHeaders(t *testing.T) { + headers := make(chan observedHeaders, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + headers <- captureRoutingHeaders(r) + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + + client := NewHTTPClientWithPPEEnv( + server.URL, + time.Second, + NewAccessKeyAuthorizer("production-ak"), + func() string { return "" }, + ) + err := client.SendRequestWithHeaders(context.Background(), "/api/test", nil, map[string]string{ + "Authorization": "Bearer caller-override", + "x-use-ppe": "1", + "x-tt-env": "ppe_caller_injection", + "x-schedule-vdc": "caller-vdc", + }, nil) + if err != nil { + t.Fatalf("SendRequest() error = %v", err) + } + got := <-headers + if got.authorization != "Bearer production-ak" || got.usePPE != "" || got.ppeEnv != "" || got.scheduleVDC != "" { + t.Fatalf("production headers = %#v", got) + } +} + +func TestHTTPClientProtectsConfiguredPPEHeadersFromCallerOverride(t *testing.T) { + headers := make(chan observedHeaders, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + headers <- captureRoutingHeaders(r) + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + + client := NewHTTPClientWithPPEEnv( + server.URL, + time.Second, + NewAccessKeyAuthorizer("configured-ak"), + func() string { return "ppe_configured" }, + ) + err := client.SendRequestWithHeaders(context.Background(), "/api/test", nil, map[string]string{ + "Authorization": "Bearer caller-override", + "x-use-ppe": "0", + "x-tt-env": "ppe_caller_override", + "x-schedule-vdc": "caller-vdc", + }, nil) + if err != nil { + t.Fatalf("SendRequest() error = %v", err) + } + got := <-headers + if got.authorization != "Bearer configured-ak" || got.usePPE != "1" || got.ppeEnv != "ppe_configured" || got.scheduleVDC != defaultPPEVDC { + t.Fatalf("protected headers = %#v", got) + } +} + +func TestHTTPClientMultipartRequestInjectsPPEHeaders(t *testing.T) { + headers := make(chan observedHeaders, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + headers <- captureRoutingHeaders(r) + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + + filePath := filepath.Join(t.TempDir(), "sample.txt") + if err := os.WriteFile(filePath, []byte("sample"), 0o600); err != nil { + t.Fatal(err) + } + client := NewHTTPClientWithPPEEnv( + server.URL, + time.Second, + NewAccessKeyAuthorizer("upload-ak"), + func() string { return "ppe_upload" }, + ) + err := client.SendMultipartRequest(context.Background(), "/api/upload", nil, MultipartFile{ + FieldName: "file", + Path: filePath, + }, nil) + if err != nil { + t.Fatalf("SendMultipartRequest() error = %v", err) + } + got := <-headers + if got.authorization != "Bearer upload-ak" || got.usePPE != "1" || got.ppeEnv != "ppe_upload" || got.scheduleVDC != defaultPPEVDC { + t.Fatalf("multipart headers = %#v", got) + } +} + +func TestSameOriginUsesEffectiveDefaultPorts(t *testing.T) { + httpsDefault, _ := url.Parse("https://xyq.jianying.com/api") + httpsExplicit, _ := url.Parse("https://XYQ.JIANYING.COM:443/asset") + if !sameOrigin(httpsDefault, httpsExplicit) { + t.Fatal("sameOrigin() = false for equivalent HTTPS origins") + } + + differentPort, _ := url.Parse("https://xyq.jianying.com:8443/api") + if sameOrigin(httpsDefault, differentPort) { + t.Fatal("sameOrigin() = true for different ports") + } +} + +func captureRoutingHeaders(r *http.Request) observedHeaders { + return observedHeaders{ + authorization: r.Header.Get("Authorization"), + usePPE: r.Header.Get(ppeUseHeader), + ppeEnv: r.Header.Get(ppeEnvHeader), + scheduleVDC: r.Header.Get(ppeScheduleVDCHeader), + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 48b34e4..2d3371c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,7 +1,9 @@ package config import ( + "fmt" "os" + "regexp" "strings" "time" ) @@ -18,8 +20,11 @@ const ( UploadFilePath = "/api/biz/v1/skill/upload_file" ListThreadFilePath = "/api/biz/v1/skill/list_thread_file" EnvXYQAccessKey = "XYQ_ACCESS_KEY" + EnvPPEEnv = "PIPPIT_CLI_PPE_ENV" ) +var ppeEnvPattern = regexp.MustCompile(`^ppe_[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + // Config holds runtime settings selected by the root command and passed down // into lower layers. type Config struct { @@ -27,6 +32,7 @@ type Config struct { HTTPTimeout time.Duration AuthTTL time.Duration AccessKey string + PPEEnv string OAuth *OAuth Paths *Paths } @@ -52,6 +58,7 @@ func Load() *Config { HTTPTimeout: DefaultHTTPTimeout, AuthTTL: DefaultAuthTTL, AccessKey: strings.TrimSpace(os.Getenv(EnvXYQAccessKey)), + PPEEnv: strings.TrimSpace(os.Getenv(EnvPPEEnv)), OAuth: resolveOAuth(), Paths: &Paths{ SubmitRun: SubmitRunPath, @@ -62,6 +69,20 @@ func Load() *Config { } } +// NormalizePPEEnv validates an optional PPE lane name before it can be sent in +// request headers. Empty selects production. Keeping the accepted character +// set narrow also prevents malformed header values from reaching net/http. +func NormalizePPEEnv(value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", nil + } + if !ppeEnvPattern.MatchString(value) { + return "", fmt.Errorf("PPE 环境 %q 非法：必须以 ppe_ 开头，且只能包含字母、数字、点、下划线或连字符", value) + } + return value, nil +} + func resolveOAuth() *OAuth { return &OAuth{ ClientKey: DefaultOAuthClientKey, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6c9d91b..561e1af 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -4,6 +4,7 @@ import "testing" func TestLoadUsesDefaultConfig(t *testing.T) { t.Setenv(EnvXYQAccessKey, "") + t.Setenv(EnvPPEEnv, "") cfg := Load() if cfg.BaseURL != DefaultBaseURL { t.Fatalf("BaseURL = %q, want %q", cfg.BaseURL, DefaultBaseURL) @@ -17,6 +18,9 @@ func TestLoadUsesDefaultConfig(t *testing.T) { if cfg.AccessKey != "" { t.Fatalf("AccessKey = %q, want empty", cfg.AccessKey) } + if cfg.PPEEnv != "" { + t.Fatalf("PPEEnv = %q, want empty", cfg.PPEEnv) + } if cfg.OAuth.ClientKey != DefaultOAuthClientKey { t.Fatalf("OAuth.ClientKey = %q, want %q", cfg.OAuth.ClientKey, DefaultOAuthClientKey) } @@ -47,3 +51,44 @@ func TestLoadReadsAccessKey(t *testing.T) { t.Fatalf("AccessKey = %q, want trimmed token", cfg.AccessKey) } } + +func TestLoadReadsPPEEnv(t *testing.T) { + t.Setenv(EnvPPEEnv, " ppe_cli_canvas_ak ") + cfg := Load() + if cfg.PPEEnv != "ppe_cli_canvas_ak" { + t.Fatalf("PPEEnv = %q, want trimmed PPE lane", cfg.PPEEnv) + } +} + +func TestNormalizePPEEnv(t *testing.T) { + tests := []struct { + name string + value string + want string + wantErr bool + }{ + {name: "production", value: "", want: ""}, + {name: "trimmed", value: " ppe_cli-canvas.1 ", want: "ppe_cli-canvas.1"}, + {name: "missing prefix", value: "cli_canvas", wantErr: true}, + {name: "header injection", value: "ppe_canvas\r\nx-evil: 1", wantErr: true}, + {name: "space", value: "ppe_canvas lane", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NormalizePPEEnv(tt.value) + if tt.wantErr { + if err == nil { + t.Fatalf("NormalizePPEEnv(%q) error = nil, want error", tt.value) + } + return + } + if err != nil { + t.Fatalf("NormalizePPEEnv(%q) error = %v", tt.value, err) + } + if got != tt.want { + t.Fatalf("NormalizePPEEnv(%q) = %q, want %q", tt.value, got, tt.want) + } + }) + } +} From 5b5f17335b32fe8724f784e1879dd8d82cf23b93 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" Date: Tue, 11 Aug 2026 15:16:15 +0800 Subject: [PATCH 02/48] feat: add personal canvas create and read core Co-authored-by: Codex --- internal/canvas/allocate.go | 69 +++++++ internal/canvas/create.go | 380 ++++++++++++++++++++++++++++++++++++ internal/canvas/get.go | 156 +++++++++++++++ internal/canvas/types.go | 68 +++++++ 4 files changed, 673 insertions(+) create mode 100644 internal/canvas/allocate.go create mode 100644 internal/canvas/create.go create mode 100644 internal/canvas/get.go create mode 100644 internal/canvas/types.go diff --git a/internal/canvas/allocate.go b/internal/canvas/allocate.go new file mode 100644 index 0000000..95eee07 --- /dev/null +++ b/internal/canvas/allocate.go @@ -0,0 +1,69 @@ +package canvas + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +const MaxAllocateCount = 5000 + +type AllocateResult struct { + AssetIDs []string `json:"asset_ids"` + LogID string `json:"log_id,omitempty"` +} + +type allocateRequest struct { + Count int `json:"count"` + Base map[string]any `json:"Base"` +} + +func Allocate(ctx context.Context, count int, runner *common.Runner) (*AllocateResult, error) { + client, err := runnerClient(runner, "canvas allocate") + if err != nil { + return nil, err + } + if count <= 0 || count > MaxAllocateCount { + return nil, fmt.Errorf("canvas allocate count must be between 1 and %d", MaxAllocateCount) + } + var envelope responseEnvelope + if err := client.SendRequest(ctx, AllocatePath, allocateRequest{Count: count, Base: base()}, &envelope); err != nil { + return nil, fmt.Errorf("canvas allocate request failed: %w", err) + } + if err := envelope.validate("canvas allocate"); err != nil { + return nil, err + } + var data map[string]json.RawMessage + if err := json.Unmarshal(envelope.Data, &data); err != nil { + return nil, common.NewLogIDError(fmt.Sprintf("canvas allocate returned invalid data: %v", err), envelope.LogID) + } + idsRaw, ok := rawField(data, "ids", "IDs", "asset_ids") + if !ok { + return nil, common.NewLogIDError("canvas allocate response is missing data.ids", envelope.LogID) + } + var ids []string + if err := json.Unmarshal(idsRaw, &ids); err != nil { + return nil, common.NewLogIDError("canvas allocate data.ids must contain JSON strings", envelope.LogID) + } + if len(ids) != count { + return nil, common.NewLogIDError( + fmt.Sprintf("canvas allocate returned %d ids, want %d", len(ids), count), + envelope.LogID, + ) + } + seen := make(map[string]struct{}, len(ids)) + for index, id := range ids { + ids[index] = strings.TrimSpace(id) + if ids[index] == "" { + return nil, common.NewLogIDError(fmt.Sprintf("canvas allocate returned empty id at index %d", index), envelope.LogID) + } + if _, duplicate := seen[ids[index]]; duplicate { + return nil, common.NewLogIDError(fmt.Sprintf("canvas allocate returned duplicate id %q", ids[index]), envelope.LogID) + } + seen[ids[index]] = struct{}{} + } + return &AllocateResult{AssetIDs: ids, LogID: strings.TrimSpace(envelope.LogID)}, nil +} diff --git a/internal/canvas/create.go b/internal/canvas/create.go new file mode 100644 index 0000000..ee5ef3d --- /dev/null +++ b/internal/canvas/create.go @@ -0,0 +1,380 @@ +package canvas + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + "time" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +const defaultCreationPollInterval = time.Second + +var terminalCreationStates = map[string]struct{}{ + "4": {}, "5": {}, "9": {}, "failed": {}, "canceled": {}, "hitl_interrupt": {}, +} + +type CreateOptions struct { + Title string + RequestID string + Wait bool + PollInterval time.Duration + WaitTimeout time.Duration +} + +type CreateResult struct { + RequestID string `json:"request_id"` + State string `json:"state"` + ProjectID string `json:"project_id"` + ThreadID string `json:"thread_id"` + RunID string `json:"run_id"` + CanvasAssetID string `json:"canvas_asset_id"` + OverviewPippitAssetID string `json:"overview_pippit_asset_id,omitempty"` + WebURL string `json:"web_url"` + LogID string `json:"log_id,omitempty"` + PollAttempts int `json:"poll_attempts,omitempty"` + Warning string `json:"warning,omitempty"` +} + +type CreationTerminalError struct { + State string +} + +func (e *CreationTerminalError) Error() string { + return fmt.Sprintf("canvas creation reached terminal state %q", strings.TrimSpace(e.State)) +} + +type createRequest struct { + Surface string `json:"surface"` + Title string `json:"title,omitempty"` + RequestID string `json:"request_id"` + Base map[string]any `json:"Base"` +} + +type createData struct { + State string `json:"state"` + ProjectID string `json:"project_id"` + ThreadID string `json:"thread_id"` + RunID string `json:"run_id"` + CanvasAssetID string `json:"canvas_asset_id"` + OverviewPippitAssetID string `json:"overview_pippit_asset_id"` + WebURL string `json:"web_url"` +} + +func Create(ctx context.Context, opts CreateOptions, runner *common.Runner) (*CreateResult, error) { + client, err := runnerClient(runner, "canvas create") + if err != nil { + return nil, err + } + if opts.PollInterval < 0 || opts.WaitTimeout < 0 { + return nil, fmt.Errorf("canvas create polling durations must not be negative") + } + requestID := strings.TrimSpace(opts.RequestID) + if requestID == "" { + return nil, fmt.Errorf("canvas create request_id is required") + } + if len([]rune(requestID)) > 128 { + return nil, fmt.Errorf("canvas create request_id must not exceed 128 characters") + } + title := strings.TrimSpace(opts.Title) + if len([]rune(title)) > 50 { + return nil, fmt.Errorf("canvas create title must not exceed 50 characters") + } + + var envelope responseEnvelope + err = client.SendRequest(ctx, CreatePath, createRequest{ + Surface: SurfaceNovel, + Title: title, + RequestID: requestID, + Base: base(), + }, &envelope) + if err != nil { + return nil, fmt.Errorf("canvas create request failed; outcome may be ambiguous, do not retry blindly: %w", err) + } + if err := envelope.validate("canvas create"); err != nil { + return nil, err + } + var data createData + if err := json.Unmarshal(envelope.Data, &data); err != nil { + return nil, common.NewLogIDError(fmt.Sprintf("canvas create returned invalid data: %v", err), envelope.LogID) + } + result := &CreateResult{ + RequestID: requestID, + State: strings.TrimSpace(data.State), + ProjectID: strings.TrimSpace(data.ProjectID), + ThreadID: strings.TrimSpace(data.ThreadID), + RunID: strings.TrimSpace(data.RunID), + CanvasAssetID: strings.TrimSpace(data.CanvasAssetID), + OverviewPippitAssetID: strings.TrimSpace(data.OverviewPippitAssetID), + WebURL: strings.TrimSpace(data.WebURL), + LogID: strings.TrimSpace(envelope.LogID), + } + if err := validateCreateResult(result); err != nil { + return nil, common.NewLogIDError(err.Error(), envelope.LogID) + } + if !opts.Wait { + return result, nil + } + if result.OverviewPippitAssetID != "" { + return finalizeReadyCanvas(ctx, result, runner) + } + + artifact, attempts, waitErr := waitForCreationArtifact(ctx, runner, result.ThreadID, result.RunID, opts.PollInterval, opts.WaitTimeout) + result.PollAttempts = attempts + if waitErr != nil { + result.Warning = waitErr.Error() + var terminal *CreationTerminalError + if errors.As(waitErr, &terminal) { + result.State = "failed" + return result, acceptedCreationError(result, waitErr) + } + // The create response was already accepted. A timeout or transient + // get_thread failure must preserve the operation IDs and must not make + // callers assume it is safe to create another project. + return result, nil + } + if artifact.CanvasAssetID != "" && artifact.CanvasAssetID != result.CanvasAssetID { + result.State = "failed" + result.Warning = fmt.Sprintf("canvas create artifact canvas_asset_id mismatch: got %q, want %q", artifact.CanvasAssetID, result.CanvasAssetID) + return result, acceptedCreationError(result, errors.New(result.Warning)) + } + result.OverviewPippitAssetID = artifact.OverviewPippitAssetID + return finalizeReadyCanvas(ctx, result, runner) +} + +func finalizeReadyCanvas(ctx context.Context, result *CreateResult, runner *common.Runner) (*CreateResult, error) { + // Preserve the confirmed overview locator even if the final root visibility + // check is temporarily unavailable. The state remains creating until that + // check succeeds, but callers can resume with the canonical project URL. + result.WebURL = readyCanvasURL(result.WebURL, result.ProjectID, result.OverviewPippitAssetID) + if _, err := queryAssets(ctx, []string{result.CanvasAssetID}, true, runner); err != nil { + result.State = StateCreating + result.Warning = fmt.Sprintf("canvas overview is complete but the root asset is not queryable yet: %v", err) + return result, nil + } + result.State = StateReady + return result, nil +} + +func acceptedCreationError(result *CreateResult, cause error) error { + return fmt.Errorf( + "%w; accepted canvas IDs: project_id=%s thread_id=%s run_id=%s canvas_asset_id=%s; do not create again blindly", + cause, result.ProjectID, result.ThreadID, result.RunID, result.CanvasAssetID, + ) +} + +func validateCreateResult(result *CreateResult) error { + if result == nil { + return fmt.Errorf("canvas create response is empty") + } + missing := make([]string, 0, 4) + for name, value := range map[string]string{ + "project_id": result.ProjectID, "thread_id": result.ThreadID, + "run_id": result.RunID, "canvas_asset_id": result.CanvasAssetID, + } { + if value == "" { + missing = append(missing, name) + } + } + if len(missing) != 0 { + return fmt.Errorf("canvas create response is missing %s", strings.Join(missing, ", ")) + } + if result.State == "" { + return fmt.Errorf("canvas create response is missing state") + } + if result.WebURL == "" { + return fmt.Errorf("canvas create response is missing web_url") + } + return nil +} + +type creationArtifact struct { + OverviewPippitAssetID string + CanvasAssetID string +} + +func waitForCreationArtifact( + ctx context.Context, + runner *common.Runner, + threadID, runID string, + pollInterval, waitTimeout time.Duration, +) (creationArtifact, int, error) { + if pollInterval <= 0 { + pollInterval = defaultCreationPollInterval + } + if waitTimeout <= 0 { + waitTimeout = 2 * time.Minute + } + waitCtx, cancel := context.WithTimeout(ctx, waitTimeout) + defer cancel() + attempts := 0 + for { + attempts++ + result, err := common.GetThread(waitCtx, &common.GetThreadOptions{ThreadID: threadID, RunID: runID}, runner) + if err != nil { + if waitCtx.Err() != nil { + return creationArtifact{}, attempts, creationWaitError(waitCtx.Err(), waitTimeout) + } + return creationArtifact{}, attempts, fmt.Errorf("poll canvas creation: %w", err) + } + run := findRunByID(result.RawData, runID) + if run != nil { + state := normalizeState(run["state"]) + artifact := findCreationArtifact(run) + if (state == "3" || state == "completed") && artifact.OverviewPippitAssetID != "" { + return artifact, attempts, nil + } + if _, terminal := terminalCreationStates[state]; terminal { + return creationArtifact{}, attempts, &CreationTerminalError{State: state} + } + } + + timer := time.NewTimer(pollInterval) + select { + case <-waitCtx.Done(): + timer.Stop() + return creationArtifact{}, attempts, creationWaitError(waitCtx.Err(), waitTimeout) + case <-timer.C: + } + } +} + +func creationWaitError(err error, timeout time.Duration) error { + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("canvas creation artifact was not ready within %s", timeout) + } + return fmt.Errorf("canvas creation wait canceled: %w", err) +} + +func findRunByID(raw json.RawMessage, runID string) map[string]any { + var value any + if json.Unmarshal(raw, &value) != nil { + return nil + } + return walkForRun(value, runID, 0) +} + +func walkForRun(value any, runID string, depth int) map[string]any { + if depth > 14 { + return nil + } + switch typed := parseJSONValue(value).(type) { + case map[string]any: + if stringValue(firstValue(typed, "run_id", "runId")) == runID { + return typed + } + for _, child := range typed { + if found := walkForRun(child, runID, depth+1); found != nil { + return found + } + } + case []any: + for _, child := range typed { + if found := walkForRun(child, runID, depth+1); found != nil { + return found + } + } + } + return nil +} + +func findCreationArtifact(value any) creationArtifact { + artifact := creationArtifact{} + walkArtifact(parseJSONValue(value), 0, &artifact) + return artifact +} + +func walkArtifact(value any, depth int, artifact *creationArtifact) { + if depth > 14 || (artifact.OverviewPippitAssetID != "" && artifact.CanvasAssetID != "") { + return + } + switch typed := parseJSONValue(value).(type) { + case map[string]any: + if stringValue(firstValue(typed, "sub_type", "subType")) == overviewPartSubtype { + data, _ := parseJSONValue(typed["data"]).(map[string]any) + if data == nil { + data = map[string]any{} + } + artifact.OverviewPippitAssetID = firstNonEmpty( + stringValue(firstValue(data, "pippit_asset_id", "pippitAssetId")), + stringValue(firstValue(typed, "pippit_asset_id", "pippitAssetId")), + ) + artifact.CanvasAssetID = stringValue(firstValue(data, "canvas_asset_id", "canvasAssetId")) + } + for _, child := range typed { + walkArtifact(child, depth+1, artifact) + } + case []any: + for _, child := range typed { + walkArtifact(child, depth+1, artifact) + } + } +} + +func parseJSONValue(value any) any { + text, ok := value.(string) + if !ok { + return value + } + text = strings.TrimSpace(text) + if text == "" || (text[0] != '{' && text[0] != '[') { + return value + } + var parsed any + if json.Unmarshal([]byte(text), &parsed) == nil { + return parsed + } + return value +} + +func firstValue(values map[string]any, keys ...string) any { + for _, key := range keys { + if value, ok := values[key]; ok { + return value + } + } + return nil +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} + +func stringValue(value any) string { + switch typed := value.(type) { + case string: + return strings.TrimSpace(typed) + case json.Number: + return typed.String() + case float64: + return fmt.Sprintf("%.0f", typed) + default: + return "" + } +} + +func normalizeState(value any) string { + return strings.ToLower(stringValue(value)) +} + +func readyCanvasURL(raw, projectID, overviewID string) string { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + parsed, _ = url.Parse("https://xyq.jianying.com/novel/detail/canvas") + } + query := parsed.Query() + query.Set("projectId", projectID) + query.Set("overviewPippitAssetId", overviewID) + query.Del("canvasId") + parsed.RawQuery = query.Encode() + return parsed.String() +} diff --git a/internal/canvas/get.go b/internal/canvas/get.go new file mode 100644 index 0000000..13473e5 --- /dev/null +++ b/internal/canvas/get.go @@ -0,0 +1,156 @@ +package canvas + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +type GetOptions struct { + AssetIDs []string +} + +type GetResult struct { + RequestedAssetIDs []string `json:"requested_asset_ids"` + Assets []json.RawMessage `json:"assets"` + LogID string `json:"log_id,omitempty"` +} + +type queryRequest struct { + PippitAssetIDs []string `json:"pippit_asset_ids"` + Base map[string]any `json:"Base"` +} + +func Get(ctx context.Context, opts GetOptions, runner *common.Runner) (*GetResult, error) { + assetIDs, err := normalizeAssetIDs(opts.AssetIDs) + if err != nil { + return nil, err + } + return queryAssets(ctx, assetIDs, true, runner) +} + +func queryAssets(ctx context.Context, assetIDs []string, requireAll bool, runner *common.Runner) (*GetResult, error) { + client, err := runnerClient(runner, "canvas get") + if err != nil { + return nil, err + } + var envelope responseEnvelope + if err := client.SendRequest(ctx, QueryPath, queryRequest{PippitAssetIDs: assetIDs, Base: base()}, &envelope); err != nil { + return nil, fmt.Errorf("canvas get request failed: %w", err) + } + if err := envelope.validate("canvas get"); err != nil { + return nil, err + } + assets, err := assetsFromData(envelope.Data) + if err != nil { + return nil, common.NewLogIDError(fmt.Sprintf("canvas get returned invalid data: %v", err), envelope.LogID) + } + + byID := make(map[string]json.RawMessage, len(assets)) + for _, asset := range assets { + assetID, err := assetIDFromRaw(asset) + if err != nil { + return nil, common.NewLogIDError(fmt.Sprintf("canvas get returned invalid asset: %v", err), envelope.LogID) + } + if _, duplicate := byID[assetID]; duplicate { + return nil, common.NewLogIDError(fmt.Sprintf("canvas get returned duplicate asset %q", assetID), envelope.LogID) + } + byID[assetID] = asset + } + + ordered := make([]json.RawMessage, 0, len(assetIDs)) + missing := make([]string, 0) + for _, assetID := range assetIDs { + asset, ok := byID[assetID] + if !ok { + missing = append(missing, assetID) + continue + } + ordered = append(ordered, asset) + } + if requireAll && len(missing) != 0 { + return nil, common.NewLogIDError( + fmt.Sprintf("canvas get did not return requested assets: %s", strings.Join(missing, ", ")), + envelope.LogID, + ) + } + return &GetResult{ + RequestedAssetIDs: append([]string(nil), assetIDs...), + Assets: ordered, + LogID: strings.TrimSpace(envelope.LogID), + }, nil +} + +func normalizeAssetIDs(values []string) ([]string, error) { + if len(values) == 0 { + return nil, fmt.Errorf("at least one canvas asset_id is required") + } + result := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for index, value := range values { + value = strings.TrimSpace(value) + if value == "" { + return nil, fmt.Errorf("canvas asset_id at index %d is empty", index) + } + if _, duplicate := seen[value]; duplicate { + return nil, fmt.Errorf("canvas asset_id %q is duplicated", value) + } + seen[value] = struct{}{} + result = append(result, value) + } + return result, nil +} + +func assetsFromData(raw json.RawMessage) ([]json.RawMessage, error) { + var data map[string]json.RawMessage + if len(raw) == 0 || string(raw) == "null" { + return nil, fmt.Errorf("data is missing") + } + if err := json.Unmarshal(raw, &data); err != nil { + return nil, err + } + assetsRaw, ok := rawField(data, "Assets", "assets") + if !ok { + return nil, fmt.Errorf("data.assets is missing") + } + var assets []json.RawMessage + if err := json.Unmarshal(assetsRaw, &assets); err != nil { + return nil, fmt.Errorf("decode data.assets: %w", err) + } + if assets == nil { + assets = []json.RawMessage{} + } + return assets, nil +} + +func assetIDFromRaw(raw json.RawMessage) (string, error) { + var asset map[string]json.RawMessage + if err := json.Unmarshal(raw, &asset); err != nil { + return "", err + } + value, ok := rawField(asset, "PippitAssetID", "pippit_asset_id", "pippitAssetId") + if !ok { + return "", fmt.Errorf("asset is missing pippit_asset_id") + } + var assetID string + if err := json.Unmarshal(value, &assetID); err != nil { + return "", fmt.Errorf("pippit_asset_id must be a JSON string") + } + assetID = strings.TrimSpace(assetID) + if assetID == "" { + return "", fmt.Errorf("pippit_asset_id is empty") + } + return assetID, nil +} + +func rawField(values map[string]json.RawMessage, keys ...string) (json.RawMessage, bool) { + for _, key := range keys { + if value, ok := values[key]; ok { + return value, true + } + } + return nil, false +} diff --git a/internal/canvas/types.go b/internal/canvas/types.go new file mode 100644 index 0000000..6ffb5a3 --- /dev/null +++ b/internal/canvas/types.go @@ -0,0 +1,68 @@ +package canvas + +import ( + "encoding/json" + "fmt" + "net/url" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +const ( + CreatePath = "/api/biz/v1/skill/canvas/create" + QueryPath = "/api/biz/v1/skill/canvas/query" + AllocatePath = "/api/biz/v1/skill/canvas/batch_generate_asset_id" + ApplyPath = "/api/biz/v1/skill/canvas/batch_patch_asset" + UploadPath = "/api/biz/v1/skill/upload_file" +) + +const ( + SurfaceNovel = "novel" + StateCreating = "creating" + StateProcessing = "processing" + StateReady = "ready" + overviewPartSubtype = "biz/x_data_novel_script_overview" +) + +type responseEnvelope struct { + Ret string `json:"ret"` + Errmsg string `json:"errmsg"` + LogID string `json:"log_id"` + Data json.RawMessage `json:"data"` +} + +func (r responseEnvelope) validate(operation string) error { + if r.Ret == "0" { + return nil + } + message := strings.TrimSpace(r.Errmsg) + if message == "" { + message = "unknown error" + } + return common.NewLogIDError( + fmt.Sprintf("%s failed: ret=%q errmsg=%s", operation, r.Ret, message), + r.LogID, + ) +} + +func base() map[string]any { + return map[string]any{} +} + +func runnerClient(runner *common.Runner, operation string) (common.Client, error) { + if runner == nil || runner.Client == nil { + return nil, fmt.Errorf("%s client is missing", operation) + } + return runner.Client, nil +} + +func pathWithProjectID(path, projectID string) string { + projectID = strings.TrimSpace(projectID) + if projectID == "" { + return path + } + query := url.Values{} + query.Set("project_id", projectID) + return path + "?" + query.Encode() +} From b904586265496f90ce461ecdb7830235e36dbca9 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" Date: Tue, 11 Aug 2026 15:17:28 +0800 Subject: [PATCH 03/48] feat: add canvas apply and upload core Co-authored-by: Codex --- internal/canvas/apply.go | 203 +++++++++++++++ internal/canvas/canvas_test.go | 446 +++++++++++++++++++++++++++++++++ internal/canvas/upload.go | 250 ++++++++++++++++++ 3 files changed, 899 insertions(+) create mode 100644 internal/canvas/apply.go create mode 100644 internal/canvas/canvas_test.go create mode 100644 internal/canvas/upload.go diff --git a/internal/canvas/apply.go b/internal/canvas/apply.go new file mode 100644 index 0000000..de53a25 --- /dev/null +++ b/internal/canvas/apply.go @@ -0,0 +1,203 @@ +package canvas + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +var decimalIDPattern = regexp.MustCompile(`^[1-9][0-9]*$`) + +type ApplyOptions struct { + ProjectID string + Request ApplyRequest +} + +type ApplyRequest struct { + BatchID string `json:"batch_id"` + ClientID string `json:"client_id"` + RootPippitAssetID string `json:"root_pippit_asset_id,omitempty"` + Transactions []PatchTransaction `json:"transactions"` + Base map[string]any `json:"Base"` +} + +type PatchTransaction struct { + TransactionID string `json:"transaction_id"` + Intent string `json:"intent,omitempty"` + MergeKey string `json:"merge_key,omitempty"` + Attempt *int32 `json:"attempt,omitempty"` + EnqueuedAt *int64 `json:"enqueued_at,omitempty"` + Patches []PatchEntry `json:"patches"` +} + +type PatchEntry struct { + AssetID string `json:"asset_id"` + BaseAssetVersion *int64 `json:"base_asset_version,omitempty"` + AssetSourceType *int32 `json:"asset_source_type,omitempty"` + Op string `json:"op"` + Path string `json:"path"` + Value json.RawMessage `json:"value,omitempty"` +} + +type ApplyResult struct { + BatchID string `json:"batch_id"` + Results []PatchTransactionResult `json:"results"` + LogID string `json:"log_id,omitempty"` +} + +type PatchTransactionResult struct { + TransactionID string `json:"transaction_id"` + Status string `json:"status"` + AssetVersions map[string]int64 `json:"asset_versions,omitempty"` + BlockedByTransaction string `json:"blocked_by,omitempty"` + Error string `json:"error,omitempty"` +} + +type applyData struct { + Results []PatchTransactionResult `json:"results"` +} + +func Apply(ctx context.Context, opts ApplyOptions, runner *common.Runner) (*ApplyResult, error) { + client, err := runnerClient(runner, "canvas apply") + if err != nil { + return nil, err + } + projectID := strings.TrimSpace(opts.ProjectID) + if projectID != "" && !decimalIDPattern.MatchString(projectID) { + return nil, fmt.Errorf("canvas apply project_id must be a positive decimal JSON string") + } + request := opts.Request + request.Base = base() + if err := validateApplyRequest(&request); err != nil { + return nil, err + } + + var envelope responseEnvelope + if err := client.SendRequest(ctx, pathWithProjectID(ApplyPath, projectID), request, &envelope); err != nil { + return nil, fmt.Errorf("canvas apply request failed; outcome may be ambiguous, do not replay blindly: %w", err) + } + if err := envelope.validate("canvas apply"); err != nil { + return nil, err + } + var data applyData + if err := json.Unmarshal(envelope.Data, &data); err != nil { + return nil, common.NewLogIDError( + fmt.Sprintf("canvas apply returned invalid data: %v; query affected assets before retrying because outcome cannot be confirmed", err), + envelope.LogID, + ) + } + ordered, err := validateApplyResults(request.Transactions, data.Results) + if err != nil { + return nil, common.NewLogIDError( + fmt.Sprintf("%s; query affected assets before retrying because outcome cannot be confirmed", err), + envelope.LogID, + ) + } + return &ApplyResult{ + BatchID: request.BatchID, + Results: ordered, + LogID: strings.TrimSpace(envelope.LogID), + }, nil +} + +func validateApplyRequest(request *ApplyRequest) error { + if request == nil { + return fmt.Errorf("canvas apply request is required") + } + request.BatchID = strings.TrimSpace(request.BatchID) + request.ClientID = strings.TrimSpace(request.ClientID) + request.RootPippitAssetID = strings.TrimSpace(request.RootPippitAssetID) + if request.BatchID == "" { + return fmt.Errorf("canvas apply batch_id is required") + } + if request.ClientID == "" { + return fmt.Errorf("canvas apply client_id is required") + } + if len(request.Transactions) != 1 { + return fmt.Errorf("canvas apply requires exactly one transaction in this beta; put related patches in that transaction") + } + transactionIDs := make(map[string]struct{}, len(request.Transactions)) + for txIndex := range request.Transactions { + tx := &request.Transactions[txIndex] + tx.TransactionID = strings.TrimSpace(tx.TransactionID) + if tx.TransactionID == "" { + return fmt.Errorf("canvas apply transactions[%d].transaction_id is required", txIndex) + } + if _, duplicate := transactionIDs[tx.TransactionID]; duplicate { + return fmt.Errorf("canvas apply transaction_id %q is duplicated", tx.TransactionID) + } + transactionIDs[tx.TransactionID] = struct{}{} + if len(tx.Patches) == 0 { + return fmt.Errorf("canvas apply transaction %q has no patches", tx.TransactionID) + } + for patchIndex := range tx.Patches { + patch := &tx.Patches[patchIndex] + patch.AssetID = strings.TrimSpace(patch.AssetID) + patch.Op = strings.ToLower(strings.TrimSpace(patch.Op)) + if patch.AssetID == "" { + return fmt.Errorf("canvas apply transaction %q patch[%d].asset_id is required", tx.TransactionID, patchIndex) + } + switch patch.Op { + case "add", "replace": + if len(patch.Value) == 0 || !json.Valid(patch.Value) { + return fmt.Errorf("canvas apply transaction %q patch[%d].value must be valid JSON", tx.TransactionID, patchIndex) + } + case "remove": + if len(patch.Value) != 0 && !json.Valid(patch.Value) { + return fmt.Errorf("canvas apply transaction %q patch[%d].value must be valid JSON", tx.TransactionID, patchIndex) + } + default: + return fmt.Errorf("canvas apply transaction %q patch[%d].op must be add, replace, or remove", tx.TransactionID, patchIndex) + } + if patch.BaseAssetVersion != nil && *patch.BaseAssetVersion < 0 { + return fmt.Errorf("canvas apply transaction %q patch[%d].base_asset_version must not be negative", tx.TransactionID, patchIndex) + } + } + } + return nil +} + +func validateApplyResults(expected []PatchTransaction, actual []PatchTransactionResult) ([]PatchTransactionResult, error) { + byID := make(map[string]PatchTransactionResult, len(actual)) + for _, result := range actual { + result.TransactionID = strings.TrimSpace(result.TransactionID) + if result.TransactionID == "" { + return nil, fmt.Errorf("canvas apply returned a result without transaction_id") + } + if _, duplicate := byID[result.TransactionID]; duplicate { + return nil, fmt.Errorf("canvas apply returned duplicate result for transaction %q", result.TransactionID) + } + byID[result.TransactionID] = result + } + if len(byID) != len(expected) { + return nil, fmt.Errorf("canvas apply returned %d transaction results, want %d", len(byID), len(expected)) + } + ordered := make([]PatchTransactionResult, 0, len(expected)) + for _, transaction := range expected { + result, ok := byID[transaction.TransactionID] + if !ok { + return nil, fmt.Errorf("canvas apply omitted transaction result %q", transaction.TransactionID) + } + if strings.ToLower(strings.TrimSpace(result.Status)) != "ack" { + return nil, fmt.Errorf( + "canvas apply transaction %q was not acknowledged: status=%q blocked_by=%q error=%q", + transaction.TransactionID, result.Status, result.BlockedByTransaction, result.Error, + ) + } + for _, patch := range transaction.Patches { + version, ok := result.AssetVersions[patch.AssetID] + if !ok { + return nil, fmt.Errorf("canvas apply transaction %q omitted version for asset %q", transaction.TransactionID, patch.AssetID) + } + if version < 0 { + return nil, fmt.Errorf("canvas apply transaction %q returned negative version for asset %q", transaction.TransactionID, patch.AssetID) + } + } + ordered = append(ordered, result) + } + return ordered, nil +} diff --git a/internal/canvas/canvas_test.go b/internal/canvas/canvas_test.go new file mode 100644 index 0000000..60c3270 --- /dev/null +++ b/internal/canvas/canvas_test.go @@ -0,0 +1,446 @@ +package canvas + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +type fakeClient struct { + send func(context.Context, string, any, any) error + multipart func(context.Context, string, map[string]string, common.MultipartFile, any) error +} + +func (f *fakeClient) SendRequest(ctx context.Context, path string, body any, out any) error { + if f.send == nil { + return fmt.Errorf("unexpected request to %s", path) + } + return f.send(ctx, path, body, out) +} + +func (f *fakeClient) SendRequestWithHeaders(ctx context.Context, path string, body any, _ map[string]string, out any) error { + return f.SendRequest(ctx, path, body, out) +} + +func (f *fakeClient) SendMultipartRequest( + ctx context.Context, + path string, + fields map[string]string, + file common.MultipartFile, + out any, +) error { + if f.multipart == nil { + return fmt.Errorf("unexpected multipart request to %s", path) + } + return f.multipart(ctx, path, fields, file, out) +} + +func runnerWithClient(client common.Client) *common.Runner { + return &common.Runner{Client: client} +} + +func decodeInto(out any, payload string) error { + return json.Unmarshal([]byte(payload), out) +} + +func requestJSON(t *testing.T, body any) map[string]any { + t.Helper() + payload, err := json.Marshal(body) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + var result map[string]any + if err := json.Unmarshal(payload, &result); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + return result +} + +func TestCreateReturnsRecoverableOperationIDs(t *testing.T) { + client := &fakeClient{send: func(_ context.Context, path string, body any, out any) error { + if path != CreatePath { + t.Fatalf("path = %q, want %q", path, CreatePath) + } + request := requestJSON(t, body) + if request["surface"] != SurfaceNovel || request["request_id"] != "request-1" { + t.Fatalf("request = %#v, want novel request", request) + } + if baseValue, ok := request["Base"].(map[string]any); !ok || len(baseValue) != 0 { + t.Fatalf("Base = %#v, want empty object", request["Base"]) + } + if _, exists := request["team_id"]; exists { + t.Fatalf("request = %#v, must not contain team context", request) + } + return decodeInto(out, `{"ret":"0","log_id":"log-create","data":{"state":"creating","project_id":"100","thread_id":"thread-1","run_id":"run-1","canvas_asset_id":"200","web_url":"https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200"}}`) + }} + + result, err := Create(context.Background(), CreateOptions{Title: "A", RequestID: "request-1"}, runnerWithClient(client)) + if err != nil { + t.Fatalf("Create() error = %v", err) + } + if result.RequestID != "request-1" || result.State != StateCreating || result.ProjectID != "100" || result.CanvasAssetID != "200" { + t.Fatalf("Create() = %#v, want recoverable creating IDs", result) + } +} + +func TestCreateWaitWithInlineOverviewStillRequiresQueryableRoot(t *testing.T) { + queryCalls := 0 + client := &fakeClient{send: func(_ context.Context, path string, _ any, out any) error { + switch path { + case CreatePath: + return decodeInto(out, `{"ret":"0","data":{"state":"creating","project_id":"100","thread_id":"thread-1","run_id":"run-1","canvas_asset_id":"200","overview_pippit_asset_id":"300","web_url":"https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200"}}`) + case QueryPath: + queryCalls++ + return decodeInto(out, `{"ret":"0","data":{"Assets":[]}}`) + default: + return fmt.Errorf("unexpected path %s", path) + } + }} + result, err := Create(context.Background(), CreateOptions{RequestID: "request-1", Wait: true}, runnerWithClient(client)) + if err != nil { + t.Fatalf("Create() error = %v", err) + } + if queryCalls != 1 || result.State != StateCreating || !strings.Contains(result.Warning, "root asset is not queryable yet") { + t.Fatalf("Create() = %#v, query calls=%d, want visibility-gated ready", result, queryCalls) + } +} + +func TestCreateTransportFailureExplainsAmbiguousOutcome(t *testing.T) { + client := &fakeClient{send: func(context.Context, string, any, any) error { + return errors.New("timeout") + }} + _, err := Create(context.Background(), CreateOptions{RequestID: "request-1"}, runnerWithClient(client)) + if err == nil || !strings.Contains(err.Error(), "outcome may be ambiguous, do not retry blindly") { + t.Fatalf("Create() error = %v, want ambiguous outcome guidance", err) + } +} + +func TestCreateWaitsForMachineArtifactWithoutV2(t *testing.T) { + getThreadCalls := 0 + client := &fakeClient{send: func(_ context.Context, path string, body any, out any) error { + switch path { + case CreatePath: + return decodeInto(out, `{"ret":"0","data":{"state":"creating","project_id":"100","thread_id":"thread-1","run_id":"run-1","canvas_asset_id":"200","web_url":"https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200"}}`) + case "/api/biz/v1/skill/get_thread": + getThreadCalls++ + request := requestJSON(t, body) + if _, exists := request["version"]; exists { + t.Fatalf("get_thread request = %#v, must not request v2 readable text", request) + } + return decodeInto(out, `{"ret":"0","data":{"thread":{"run_list":[{"run_id":"run-1","state":3,"content":[{"sub_type":"biz/x_data_novel_script_overview","data":"{\"pippit_asset_id\":\"300\",\"canvas_asset_id\":\"200\"}"}] }]}}}`) + case QueryPath: + return decodeInto(out, `{"ret":"0","data":{"Assets":[{"PippitAssetID":"200"}]}}`) + default: + return fmt.Errorf("unexpected path %s", path) + } + }} + + result, err := Create(context.Background(), CreateOptions{ + RequestID: "request-1", + Wait: true, + PollInterval: time.Millisecond, + WaitTimeout: time.Second, + }, runnerWithClient(client)) + if err != nil { + t.Fatalf("Create() error = %v", err) + } + if getThreadCalls != 1 || result.State != StateReady || result.OverviewPippitAssetID != "300" { + t.Fatalf("Create() = %#v, calls=%d, want ready artifact", result, getThreadCalls) + } + if strings.Contains(result.WebURL, "canvasId=") || !strings.Contains(result.WebURL, "overviewPippitAssetId=300") { + t.Fatalf("WebURL = %q, want canonical overview URL", result.WebURL) + } +} + +func TestCreateKeepsCreatingWhenRootAssetIsNotQueryable(t *testing.T) { + client := &fakeClient{send: func(_ context.Context, path string, _ any, out any) error { + switch path { + case CreatePath: + return decodeInto(out, `{"ret":"0","data":{"state":"creating","project_id":"100","thread_id":"thread-1","run_id":"run-1","canvas_asset_id":"200","web_url":"https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200"}}`) + case "/api/biz/v1/skill/get_thread": + return decodeInto(out, `{"ret":"0","data":{"thread":{"run_list":[{"run_id":"run-1","state":3,"content":[{"sub_type":"biz/x_data_novel_script_overview","data":"{\"pippit_asset_id\":\"300\",\"canvas_asset_id\":\"200\"}"}]}]}}}`) + case QueryPath: + return decodeInto(out, `{"ret":"0","log_id":"query-log","data":{"Assets":[]}}`) + default: + return fmt.Errorf("unexpected path %s", path) + } + }} + result, err := Create(context.Background(), CreateOptions{ + RequestID: "request-1", Wait: true, PollInterval: time.Millisecond, WaitTimeout: time.Second, + }, runnerWithClient(client)) + if err != nil { + t.Fatalf("Create() error = %v, want accepted processing result", err) + } + if result.State != StateCreating || result.ProjectID != "100" || result.OverviewPippitAssetID != "300" || !strings.Contains(result.Warning, "root asset is not queryable yet") { + t.Fatalf("Create() = %#v, want creating state with query warning", result) + } + if strings.Contains(result.WebURL, "canvasId=") || !strings.Contains(result.WebURL, "overviewPippitAssetId=300") { + t.Fatalf("WebURL = %q, want preserved canonical overview URL", result.WebURL) + } +} + +func TestCreateWaitTimeoutPreservesAcceptedIDs(t *testing.T) { + client := &fakeClient{send: func(_ context.Context, path string, _ any, out any) error { + switch path { + case CreatePath: + return decodeInto(out, `{"ret":"0","data":{"state":"creating","project_id":"100","thread_id":"thread-1","run_id":"run-1","canvas_asset_id":"200","web_url":"https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200"}}`) + case "/api/biz/v1/skill/get_thread": + return decodeInto(out, `{"ret":"0","data":{"thread":{"run_list":[{"run_id":"run-1","state":1}]}}}`) + default: + return fmt.Errorf("unexpected path %s", path) + } + }} + + result, err := Create(context.Background(), CreateOptions{ + RequestID: "request-1", + Wait: true, + PollInterval: 50 * time.Millisecond, + WaitTimeout: time.Millisecond, + }, runnerWithClient(client)) + if err != nil { + t.Fatalf("Create() error = %v, want accepted result with warning", err) + } + if result == nil || result.ProjectID != "100" || result.ThreadID != "thread-1" || result.RunID != "run-1" || result.CanvasAssetID != "200" { + t.Fatalf("Create() = %#v, want all accepted IDs", result) + } + if result.State != StateCreating || !strings.Contains(result.Warning, "was not ready") { + t.Fatalf("Create() = %#v, want creating state with timeout warning", result) + } +} + +func TestCreateTerminalFailureReturnsAcceptedIDsAndError(t *testing.T) { + client := &fakeClient{send: func(_ context.Context, path string, _ any, out any) error { + if path == CreatePath { + return decodeInto(out, `{"ret":"0","data":{"state":"creating","project_id":"100","thread_id":"thread-1","run_id":"run-1","canvas_asset_id":"200","web_url":"https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200"}}`) + } + return decodeInto(out, `{"ret":"0","data":{"thread":{"run_list":[{"run_id":"run-1","state":4}]}}}`) + }} + result, err := Create(context.Background(), CreateOptions{ + RequestID: "request-1", Wait: true, PollInterval: time.Millisecond, WaitTimeout: time.Second, + }, runnerWithClient(client)) + if result == nil || result.ProjectID != "100" || result.State != "failed" { + t.Fatalf("Create() result = %#v, want failed result with accepted IDs", result) + } + if err == nil || !strings.Contains(err.Error(), "accepted canvas IDs") || !strings.Contains(err.Error(), "do not create again blindly") { + t.Fatalf("Create() error = %v, want accepted ID guidance", err) + } +} + +func TestGetRequiresEveryRequestedStringID(t *testing.T) { + client := &fakeClient{send: func(_ context.Context, path string, body any, out any) error { + if path != QueryPath { + t.Fatalf("path = %q, want query", path) + } + return decodeInto(out, `{"ret":"0","log_id":"log-query","data":{"Assets":[{"PippitAssetID":"2"},{"PippitAssetID":"1"}]}}`) + }} + result, err := Get(context.Background(), GetOptions{AssetIDs: []string{"1", "2"}}, runnerWithClient(client)) + if err != nil { + t.Fatalf("Get() error = %v", err) + } + firstID, _ := assetIDFromRaw(result.Assets[0]) + if firstID != "1" || result.LogID != "log-query" { + t.Fatalf("Get() = %#v, want request order and log ID", result) + } + + missingClient := &fakeClient{send: func(_ context.Context, _ string, _ any, out any) error { + return decodeInto(out, `{"ret":"0","log_id":"missing-log","data":{"Assets":[]}}`) + }} + _, err = Get(context.Background(), GetOptions{AssetIDs: []string{"1"}}, runnerWithClient(missingClient)) + if err == nil || !strings.Contains(err.Error(), "did not return requested assets: 1") || !strings.Contains(err.Error(), "missing-log") { + t.Fatalf("Get() error = %v, want strict missing asset error", err) + } +} + +func TestGetRejectsNumericAssetIDInResponse(t *testing.T) { + client := &fakeClient{send: func(_ context.Context, _ string, _ any, out any) error { + return decodeInto(out, `{"ret":"0","data":{"Assets":[{"PippitAssetID":123}]}}`) + }} + _, err := Get(context.Background(), GetOptions{AssetIDs: []string{"123"}}, runnerWithClient(client)) + if err == nil || !strings.Contains(err.Error(), "must be a JSON string") { + t.Fatalf("Get() error = %v, want string ID validation", err) + } +} + +func TestAllocateRequiresExactUniqueStringIDs(t *testing.T) { + client := &fakeClient{send: func(_ context.Context, path string, body any, out any) error { + if path != AllocatePath { + t.Fatalf("path = %q, want allocate", path) + } + request := requestJSON(t, body) + if request["count"] != float64(2) { + t.Fatalf("count = %#v, want 2", request["count"]) + } + return decodeInto(out, `{"ret":"0","data":{"ids":["10","11"]}}`) + }} + result, err := Allocate(context.Background(), 2, runnerWithClient(client)) + if err != nil || strings.Join(result.AssetIDs, ",") != "10,11" { + t.Fatalf("Allocate() = (%#v, %v), want two IDs", result, err) + } + + duplicate := &fakeClient{send: func(_ context.Context, _ string, _ any, out any) error { + return decodeInto(out, `{"ret":"0","data":{"ids":["10","10"]}}`) + }} + _, err = Allocate(context.Background(), 2, runnerWithClient(duplicate)) + if err == nil || !strings.Contains(err.Error(), "duplicate id") { + t.Fatalf("Allocate() error = %v, want duplicate rejection", err) + } +} + +func TestApplyRequiresAcknowledgementAndEveryAssetVersion(t *testing.T) { + version := int64(0) + request := ApplyRequest{ + BatchID: "batch-1", + ClientID: "client-1", + Transactions: []PatchTransaction{{ + TransactionID: "tx-1", + Patches: []PatchEntry{{AssetID: "asset-1", BaseAssetVersion: &version, Op: "add", Path: "", Value: json.RawMessage(`{"nodes":[]}`)}}, + }}, + } + client := &fakeClient{send: func(_ context.Context, path string, body any, out any) error { + if path != ApplyPath+"?project_id=123" { + t.Fatalf("path = %q, want project query", path) + } + payload := requestJSON(t, body) + if baseValue, ok := payload["Base"].(map[string]any); !ok || len(baseValue) != 0 { + t.Fatalf("Base = %#v, want empty object", payload["Base"]) + } + return decodeInto(out, `{"ret":"0","log_id":"log-apply","data":{"results":[{"transaction_id":"tx-1","status":"ack","asset_versions":{"asset-1":1}}]}}`) + }} + result, err := Apply(context.Background(), ApplyOptions{ProjectID: "123", Request: request}, runnerWithClient(client)) + if err != nil || result.Results[0].AssetVersions["asset-1"] != 1 { + t.Fatalf("Apply() = (%#v, %v), want ack", result, err) + } + + rejected := &fakeClient{send: func(_ context.Context, _ string, _ any, out any) error { + return decodeInto(out, `{"ret":"0","data":{"results":[{"transaction_id":"tx-1","status":"blocked","blocked_by":"tx-0"}]}}`) + }} + _, err = Apply(context.Background(), ApplyOptions{Request: request}, runnerWithClient(rejected)) + if err == nil || !strings.Contains(err.Error(), "was not acknowledged") { + t.Fatalf("Apply() error = %v, want status rejection", err) + } + + missingVersion := &fakeClient{send: func(_ context.Context, _ string, _ any, out any) error { + return decodeInto(out, `{"ret":"0","data":{"results":[{"transaction_id":"tx-1","status":"ack","asset_versions":{}}]}}`) + }} + _, err = Apply(context.Background(), ApplyOptions{Request: request}, runnerWithClient(missingVersion)) + if err == nil || !strings.Contains(err.Error(), "omitted version") { + t.Fatalf("Apply() error = %v, want missing version rejection", err) + } +} + +func TestApplyBetaRejectsMultipleTransactionsBeforeRequest(t *testing.T) { + requestCalls := 0 + client := &fakeClient{send: func(context.Context, string, any, any) error { + requestCalls++ + return nil + }} + request := ApplyRequest{ + BatchID: "batch-1", ClientID: "client-1", + Transactions: []PatchTransaction{ + {TransactionID: "tx-1", Patches: []PatchEntry{{AssetID: "asset-1", Op: "add", Path: "", Value: json.RawMessage(`{}`)}}}, + {TransactionID: "tx-2", Patches: []PatchEntry{{AssetID: "asset-2", Op: "add", Path: "", Value: json.RawMessage(`{}`)}}}, + }, + } + _, err := Apply(context.Background(), ApplyOptions{Request: request}, runnerWithClient(client)) + if err == nil || !strings.Contains(err.Error(), "exactly one transaction") { + t.Fatalf("Apply() error = %v, want single-transaction beta guard", err) + } + if requestCalls != 0 { + t.Fatalf("request calls = %d, want validation before side effects", requestCalls) + } +} + +func TestApplyTransportFailureWarnsAgainstBlindReplay(t *testing.T) { + request := ApplyRequest{ + BatchID: "batch-1", ClientID: "client-1", + Transactions: []PatchTransaction{{TransactionID: "tx-1", Patches: []PatchEntry{{ + AssetID: "asset-1", Op: "replace", Path: "", Value: json.RawMessage(`{}`), + }}}}, + } + client := &fakeClient{send: func(context.Context, string, any, any) error { return errors.New("timeout") }} + _, err := Apply(context.Background(), ApplyOptions{Request: request}, runnerWithClient(client)) + if err == nil || !strings.Contains(err.Error(), "do not replay blindly") { + t.Fatalf("Apply() error = %v, want replay warning", err) + } +} + +func TestUploadWaitsForQueryableAssetWithoutRequiringCover(t *testing.T) { + path := filepath.Join(t.TempDir(), "clip.mp4") + if err := os.WriteFile(path, []byte("video"), 0o600); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + client := &fakeClient{ + multipart: func(_ context.Context, gotPath string, fields map[string]string, file common.MultipartFile, out any) error { + if gotPath != UploadPath || len(fields) != 0 { + t.Fatalf("multipart = (%q, %#v), want upload without auth fields", gotPath, fields) + } + if file.FieldName != "file" || file.ContentType != "video/mp4" { + t.Fatalf("file = %#v, want video multipart", file) + } + return decodeInto(out, `{"ret":"0","log_id":"upload-log","data":{"asset_id":"workspace-1","pippit_asset_id":"asset-1"}}`) + }, + send: func(_ context.Context, path string, _ any, out any) error { + if path != QueryPath { + t.Fatalf("path = %q, want query", path) + } + return decodeInto(out, `{"ret":"0","log_id":"query-log","data":{"Assets":[{"PippitAssetID":"asset-1","SourceID":"workspace-1","Video":{"DownloadUrl":"https://example.test/clip.mp4","VID":"vid-1"}}]}}`) + }, + } + result, err := Upload(context.Background(), UploadOptions{ + Path: path, PollInterval: time.Millisecond, WaitTimeout: time.Second, + }, runnerWithClient(client)) + if err != nil { + t.Fatalf("Upload() error = %v", err) + } + if result.State != StateReady || result.AssetID != "workspace-1" || result.PippitAssetID != "asset-1" || result.PollAttempts != 1 { + t.Fatalf("Upload() = %#v, want ready identifiers", result) + } + if result.Locator.DownloadURL == "" || result.Locator.CoverURL != "" { + t.Fatalf("Locator = %#v, want playable locator without cover requirement", result.Locator) + } +} + +func TestUploadWaitTimeoutPreservesDurableAssetID(t *testing.T) { + path := filepath.Join(t.TempDir(), "clip.mp4") + if err := os.WriteFile(path, []byte("video"), 0o600); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + client := &fakeClient{ + multipart: func(_ context.Context, _ string, _ map[string]string, _ common.MultipartFile, out any) error { + return decodeInto(out, `{"ret":"0","log_id":"upload-log","data":{"asset_id":"workspace-1","pippit_asset_id":"asset-1"}}`) + }, + send: func(_ context.Context, _ string, _ any, out any) error { + return decodeInto(out, `{"ret":"0","data":{"Assets":[]}}`) + }, + } + result, err := Upload(context.Background(), UploadOptions{ + Path: path, PollInterval: 50 * time.Millisecond, WaitTimeout: time.Millisecond, + }, runnerWithClient(client)) + if err != nil { + t.Fatalf("Upload() error = %v, want accepted processing result", err) + } + if result.State != StateProcessing || result.PippitAssetID != "asset-1" || result.Locator.PippitAssetID != "asset-1" { + t.Fatalf("Upload() = %#v, want durable ID after wait timeout", result) + } + if !strings.Contains(result.Warning, "was not queryable") { + t.Fatalf("warning = %q, want visibility warning", result.Warning) + } +} + +func TestEnvelopeRetIsStrict(t *testing.T) { + client := &fakeClient{send: func(_ context.Context, _ string, _ any, out any) error { + return decodeInto(out, `{"errmsg":"missing ret","log_id":"strict-log","data":{"Assets":[]}}`) + }} + _, err := Get(context.Background(), GetOptions{AssetIDs: []string{"1"}}, runnerWithClient(client)) + if err == nil || !strings.Contains(err.Error(), `ret=""`) || !strings.Contains(err.Error(), "strict-log") { + t.Fatalf("Get() error = %v, want strict ret validation", err) + } +} diff --git a/internal/canvas/upload.go b/internal/canvas/upload.go new file mode 100644 index 0000000..482f271 --- /dev/null +++ b/internal/canvas/upload.go @@ -0,0 +1,250 @@ +package canvas + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "mime" + "os" + "path/filepath" + "strings" + "time" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +const defaultUploadPollInterval = time.Second + +var uploadContentTypeFallbacks = map[string]string{ + ".jpeg": "image/jpeg", + ".jpg": "image/jpeg", + ".png": "image/png", + ".webp": "image/webp", + ".mp3": "audio/mpeg", + ".wav": "audio/wav", + ".mp4": "video/mp4", + ".mov": "video/quicktime", +} + +type UploadOptions struct { + Path string + PollInterval time.Duration + WaitTimeout time.Duration +} + +// AssetLocator reports the durable Pippit ID and optional observed media +// locations. URLs may be signed or short lived and must not be persisted into +// Canvas patches; callers should persist PippitAssetID instead. +type AssetLocator struct { + PippitAssetID string `json:"pippit_asset_id"` + SourceID string `json:"source_id,omitempty"` + DownloadURL string `json:"download_url,omitempty"` + InternalURL string `json:"internal_url,omitempty"` + CoverURL string `json:"cover_url,omitempty"` + VID string `json:"vid,omitempty"` +} + +type UploadResult struct { + State string `json:"state"` + AssetID string `json:"asset_id,omitempty"` + PippitAssetID string `json:"pippit_asset_id"` + Locator AssetLocator `json:"locator"` + Asset json.RawMessage `json:"asset,omitempty"` + LogID string `json:"log_id,omitempty"` + QueryLogID string `json:"query_log_id,omitempty"` + PollAttempts int `json:"poll_attempts"` + Warning string `json:"warning,omitempty"` +} + +func Upload(ctx context.Context, opts UploadOptions, runner *common.Runner) (*UploadResult, error) { + client, err := runnerClient(runner, "canvas upload") + if err != nil { + return nil, err + } + if opts.PollInterval < 0 || opts.WaitTimeout < 0 { + return nil, fmt.Errorf("canvas upload polling durations must not be negative") + } + path := strings.TrimSpace(opts.Path) + if path == "" { + return nil, fmt.Errorf("canvas upload path is required") + } + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("inspect canvas upload file: %w", err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("canvas upload path %q is not a regular file", path) + } + fileName := filepath.Base(path) + extension := strings.ToLower(filepath.Ext(fileName)) + contentType := mime.TypeByExtension(extension) + if contentType == "" { + contentType = uploadContentTypeFallbacks[extension] + } + if contentType == "" { + contentType = "application/octet-stream" + } + + var envelope responseEnvelope + if err := client.SendMultipartRequest(ctx, UploadPath, nil, common.MultipartFile{ + FieldName: "file", + Path: path, + FileName: fileName, + ContentType: contentType, + }, &envelope); err != nil { + return nil, fmt.Errorf("canvas upload request failed; outcome may be ambiguous, check assets before retrying: %w", err) + } + if err := envelope.validate("canvas upload"); err != nil { + return nil, err + } + assetID, pippitAssetID, err := parseUploadData(envelope.Data) + if err != nil { + return nil, common.NewLogIDError(fmt.Sprintf("canvas upload returned invalid data: %v", err), envelope.LogID) + } + result := &UploadResult{ + State: StateProcessing, + AssetID: assetID, + PippitAssetID: pippitAssetID, + Locator: AssetLocator{PippitAssetID: pippitAssetID}, + LogID: strings.TrimSpace(envelope.LogID), + } + + asset, queryLogID, attempts, err := waitForUploadedAsset(ctx, pippitAssetID, opts.PollInterval, opts.WaitTimeout, runner) + result.PollAttempts = attempts + if err != nil { + // The upload response already returned a durable Pippit asset ID. Keep + // it machine-readable so callers can resume querying without uploading + // the same bytes again. + result.Warning = err.Error() + return result, nil + } + result.State = StateReady + result.Locator = locatorFromAsset(asset, pippitAssetID) + result.Asset = asset + result.QueryLogID = queryLogID + return result, nil +} + +func parseUploadData(raw json.RawMessage) (string, string, error) { + var data map[string]json.RawMessage + if err := json.Unmarshal(raw, &data); err != nil { + return "", "", err + } + assetID, err := optionalStringField(data, "asset_id", "AssetId", "assetId") + if err != nil { + return "", "", err + } + pippitAssetID, err := optionalStringField(data, "pippit_asset_id", "PippitAssetID", "pippitAssetId") + if err != nil { + return "", "", err + } + if pippitAssetID == "" { + return "", "", fmt.Errorf("data.pippit_asset_id is missing") + } + return assetID, pippitAssetID, nil +} + +func optionalStringField(values map[string]json.RawMessage, keys ...string) (string, error) { + raw, ok := rawField(values, keys...) + if !ok || string(raw) == "null" { + return "", nil + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return "", fmt.Errorf("%s must be a JSON string", keys[0]) + } + return strings.TrimSpace(value), nil +} + +func waitForUploadedAsset( + ctx context.Context, + assetID string, + pollInterval, waitTimeout time.Duration, + runner *common.Runner, +) (json.RawMessage, string, int, error) { + if pollInterval <= 0 { + pollInterval = defaultUploadPollInterval + } + if waitTimeout <= 0 { + waitTimeout = 2 * time.Minute + } + waitCtx, cancel := context.WithTimeout(ctx, waitTimeout) + defer cancel() + for attempts := 1; ; attempts++ { + result, err := queryAssets(waitCtx, []string{assetID}, false, runner) + if err != nil { + if waitCtx.Err() != nil { + return nil, "", attempts, uploadWaitError(waitCtx.Err(), assetID, waitTimeout) + } + return nil, "", attempts, fmt.Errorf("query uploaded canvas asset: %w", err) + } + if len(result.Assets) == 1 { + return result.Assets[0], result.LogID, attempts, nil + } + + timer := time.NewTimer(pollInterval) + select { + case <-waitCtx.Done(): + timer.Stop() + return nil, result.LogID, attempts, uploadWaitError(waitCtx.Err(), assetID, waitTimeout) + case <-timer.C: + } + } +} + +func uploadWaitError(err error, assetID string, timeout time.Duration) error { + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("uploaded canvas asset %q was not queryable within %s", assetID, timeout) + } + return fmt.Errorf("wait for uploaded canvas asset %q canceled: %w", assetID, err) +} + +func locatorFromAsset(raw json.RawMessage, pippitAssetID string) AssetLocator { + var value any + _ = json.Unmarshal(raw, &value) + return AssetLocator{ + PippitAssetID: pippitAssetID, + SourceID: findFirstString(value, "SourceID", "source_id", "sourceId"), + DownloadURL: findFirstString(value, "DownloadUrl", "DownloadURL", "download_url", "downloadUrl"), + InternalURL: findFirstString(value, "InternalUrl", "InternalURL", "internal_url", "internalUrl"), + CoverURL: findFirstString(value, "CoverUrl", "CoverURL", "cover_url", "coverUrl"), + VID: findFirstString(value, "VID", "vid"), + } +} + +func findFirstString(value any, keys ...string) string { + keySet := make(map[string]struct{}, len(keys)) + for _, key := range keys { + keySet[key] = struct{}{} + } + return walkFirstString(value, keySet, 0) +} + +func walkFirstString(value any, keys map[string]struct{}, depth int) string { + if depth > 10 { + return "" + } + switch typed := value.(type) { + case map[string]any: + for key, candidate := range typed { + if _, ok := keys[key]; ok { + if text, ok := candidate.(string); ok && strings.TrimSpace(text) != "" { + return strings.TrimSpace(text) + } + } + } + for _, candidate := range typed { + if found := walkFirstString(candidate, keys, depth+1); found != "" { + return found + } + } + case []any: + for _, candidate := range typed { + if found := walkFirstString(candidate, keys, depth+1); found != "" { + return found + } + } + } + return "" +} From c64e642be6059e6429029a96a9c85bb8c3a833c6 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" Date: Tue, 11 Aug 2026 15:17:38 +0800 Subject: [PATCH 04/48] feat: expose personal canvas commands Co-authored-by: Codex --- cmd/canvas/canvas.go | 216 ++++++++++++++++++++++++++++++++++++++ cmd/canvas/canvas_test.go | 106 +++++++++++++++++++ cmd/root.go | 27 ++++- cmd/root_test.go | 100 ++++++++++++++++++ cmd/short_drama_test.go | 2 +- 5 files changed, 449 insertions(+), 2 deletions(-) create mode 100644 cmd/canvas/canvas.go create mode 100644 cmd/canvas/canvas_test.go create mode 100644 cmd/root_test.go diff --git a/cmd/canvas/canvas.go b/cmd/canvas/canvas.go new file mode 100644 index 0000000..5a86064 --- /dev/null +++ b/cmd/canvas/canvas.go @@ -0,0 +1,216 @@ +package canvas + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" + "github.com/Pippit-dev/pippit-cli/internal/common" + "github.com/spf13/cobra" +) + +const maxApplyRequestBytes = 64 << 20 + +// NewCommand builds the provider-neutral personal Canvas command tree. +// Register it from cmd/root.go with: +// +// root.AddCommand(canvas.NewCommand(stdout, stderr, runner)) +func NewCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { + cmd := &cobra.Command{ + Use: "canvas", + Short: "Create and operate personal novel canvases", + Args: cobra.NoArgs, + } + cmd.SetOut(stdout) + cmd.SetErr(stderr) + cmd.AddCommand(newCreateCommand(stdout, stderr, runner)) + cmd.AddCommand(newGetCommand(stdout, stderr, runner)) + cmd.AddCommand(newApplyCommand(stdout, stderr, runner)) + cmd.AddCommand(newUploadCommand(stdout, stderr, runner)) + return cmd +} + +func newCreateCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { + var opts canvascore.CreateOptions + cmd := &cobra.Command{ + Use: "create", + Short: "Create a personal novel Canvas project", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if strings.TrimSpace(opts.RequestID) == "" { + requestID, err := newRequestID() + if err != nil { + return fmt.Errorf("generate canvas request ID: %w", err) + } + opts.RequestID = requestID + } + result, err := canvascore.Create(cmd.Context(), opts, runner) + if result != nil { + if writeErr := common.WriteJSON(stdout, result); writeErr != nil { + return writeErr + } + } + if err != nil { + logCanvasError("canvas create", err, map[string]string{"request_id": opts.RequestID}) + return err + } + return nil + }, + } + cmd.SetOut(stdout) + cmd.SetErr(stderr) + flags := cmd.Flags() + flags.StringVar(&opts.Title, "title", "", "project title (maximum 50 characters)") + flags.StringVar(&opts.RequestID, "request-id", "", "caller request ID; generated when omitted") + flags.BoolVar(&opts.Wait, "wait", false, "wait for the novel overview artifact") + flags.DurationVar(&opts.PollInterval, "poll-interval", time.Second, "artifact polling interval") + flags.DurationVar(&opts.WaitTimeout, "timeout", 2*time.Minute, "maximum artifact wait time") + return cmd +} + +func newGetCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { + var assetIDs []string + cmd := &cobra.Command{ + Use: "get", + Short: "Get personal Canvas assets by ID", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + result, err := canvascore.Get(cmd.Context(), canvascore.GetOptions{AssetIDs: assetIDs}, runner) + if err != nil { + logCanvasError("canvas get", err, map[string]string{"asset_count": fmt.Sprint(len(assetIDs))}) + return err + } + return common.WriteJSON(stdout, result) + }, + } + cmd.SetOut(stdout) + cmd.SetErr(stderr) + cmd.Flags().StringArrayVar(&assetIDs, "asset-id", nil, "Pippit asset ID; repeat for multiple assets") + return cmd +} + +func newApplyCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { + var filePath string + var projectID string + cmd := &cobra.Command{ + Use: "apply", + Short: "Apply one Canvas patch transaction", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + request, err := readApplyRequest(cmd.InOrStdin(), filePath) + if err != nil { + return err + } + result, err := canvascore.Apply(cmd.Context(), canvascore.ApplyOptions{ + ProjectID: projectID, + Request: request, + }, runner) + if err != nil { + logCanvasError("canvas apply", err, map[string]string{ + "batch_id": request.BatchID, + "project_id": projectID, + "transactions": fmt.Sprint(len(request.Transactions)), + }) + return err + } + return common.WriteJSON(stdout, result) + }, + } + cmd.SetOut(stdout) + cmd.SetErr(stderr) + flags := cmd.Flags() + flags.StringVar(&filePath, "file", "-", "BatchPatch JSON request file, or - for stdin") + flags.StringVar(&projectID, "project-id", "", "personal novel project ID as a decimal string") + return cmd +} + +func newUploadCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { + var opts canvascore.UploadOptions + cmd := &cobra.Command{ + Use: "upload", + Short: "Upload a file to personal assets and wait until it is queryable", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + result, err := canvascore.Upload(cmd.Context(), opts, runner) + if err != nil { + logCanvasError("canvas upload", err, map[string]string{"file_name": filepath.Base(strings.TrimSpace(opts.Path))}) + return err + } + return common.WriteJSON(stdout, result) + }, + } + cmd.SetOut(stdout) + cmd.SetErr(stderr) + flags := cmd.Flags() + flags.StringVar(&opts.Path, "path", "", "local file path to upload") + flags.DurationVar(&opts.PollInterval, "poll-interval", time.Second, "asset visibility polling interval") + flags.DurationVar(&opts.WaitTimeout, "timeout", 2*time.Minute, "maximum asset visibility wait time") + return cmd +} + +func newRequestID() (string, error) { + random := make([]byte, 16) + if _, err := rand.Read(random); err != nil { + return "", err + } + return "pippit_cli_canvas_" + hex.EncodeToString(random), nil +} + +func readApplyRequest(stdin io.Reader, filePath string) (canvascore.ApplyRequest, error) { + filePath = strings.TrimSpace(filePath) + if filePath == "" { + return canvascore.ApplyRequest{}, fmt.Errorf("canvas apply --file must not be empty") + } + var reader io.Reader + var file *os.File + if filePath == "-" { + reader = stdin + } else { + var err error + file, err = os.Open(filePath) + if err != nil { + return canvascore.ApplyRequest{}, fmt.Errorf("open canvas apply request: %w", err) + } + defer file.Close() + reader = file + } + limited := io.LimitReader(reader, maxApplyRequestBytes+1) + payload, err := io.ReadAll(limited) + if err != nil { + return canvascore.ApplyRequest{}, fmt.Errorf("read canvas apply request: %w", err) + } + if len(payload) > maxApplyRequestBytes { + return canvascore.ApplyRequest{}, fmt.Errorf("canvas apply request exceeds %d bytes", maxApplyRequestBytes) + } + decoder := json.NewDecoder(strings.NewReader(string(payload))) + decoder.DisallowUnknownFields() + var request canvascore.ApplyRequest + if err := decoder.Decode(&request); err != nil { + return canvascore.ApplyRequest{}, fmt.Errorf("decode canvas apply request: %w", err) + } + if err := ensureJSONEOF(decoder); err != nil { + return canvascore.ApplyRequest{}, err + } + return request, nil +} + +func ensureJSONEOF(decoder *json.Decoder) error { + var trailing any + if err := decoder.Decode(&trailing); err == io.EOF { + return nil + } else if err != nil { + return fmt.Errorf("decode trailing canvas apply data: %w", err) + } + return fmt.Errorf("canvas apply request must contain exactly one JSON object") +} + +func logCanvasError(command string, err error, fields map[string]string) { + _ = common.AppendDailyErrorLog(command, err, fields) +} diff --git a/cmd/canvas/canvas_test.go b/cmd/canvas/canvas_test.go new file mode 100644 index 0000000..cd9636a --- /dev/null +++ b/cmd/canvas/canvas_test.go @@ -0,0 +1,106 @@ +package canvas + +import ( + "bytes" + "context" + "encoding/json" + "strings" + "testing" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +type commandFakeClient struct { + response string + request map[string]any + handler func(string, any, any) error +} + +func (f *commandFakeClient) SendRequest(_ context.Context, path string, body any, out any) error { + payload, _ := json.Marshal(body) + _ = json.Unmarshal(payload, &f.request) + if f.handler != nil { + return f.handler(path, body, out) + } + return json.Unmarshal([]byte(f.response), out) +} + +func (f *commandFakeClient) SendRequestWithHeaders(ctx context.Context, path string, body any, _ map[string]string, out any) error { + return f.SendRequest(ctx, path, body, out) +} + +func (f *commandFakeClient) SendMultipartRequest(context.Context, string, map[string]string, common.MultipartFile, any) error { + return nil +} + +func TestCommandExposesOnlyProviderNeutralPublicVerbs(t *testing.T) { + cmd := NewCommand(&bytes.Buffer{}, &bytes.Buffer{}, &common.Runner{Client: &commandFakeClient{}}) + got := make([]string, 0, len(cmd.Commands())) + for _, child := range cmd.Commands() { + got = append(got, child.Name()) + } + if strings.Join(got, ",") != "apply,create,get,upload" { + t.Fatalf("commands = %v, want apply/create/get/upload", got) + } + for _, forbidden := range []string{"import", "bind", "team", "libtv", "allocate"} { + if strings.Contains(strings.ToLower(cmd.CommandPath()+" "+cmd.Short+" "+strings.Join(got, " ")), forbidden) { + t.Fatalf("public command surface contains forbidden verb %q", forbidden) + } + } +} + +func TestCreateCommandPrintsOneMachineReadableJSONLine(t *testing.T) { + client := &commandFakeClient{response: `{"ret":"0","log_id":"log-1","data":{"state":"creating","project_id":"100","thread_id":"thread-1","run_id":"run-1","canvas_asset_id":"200","web_url":"https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200"}}`} + var stdout, stderr bytes.Buffer + cmd := NewCommand(&stdout, &stderr, &common.Runner{Client: client}) + cmd.SetArgs([]string{"create", "--title", "Demo", "--request-id", "request-1"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if strings.Count(stdout.String(), "\n") != 1 { + t.Fatalf("stdout = %q, want one JSON line", stdout.String()) + } + var result map[string]any + if err := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &result); err != nil { + t.Fatalf("stdout is not JSON: %v", err) + } + if result["request_id"] != "request-1" || result["project_id"] != "100" || client.request["surface"] != "novel" { + t.Fatalf("result/request = (%#v, %#v), want personal novel create", result, client.request) + } +} + +func TestCreateCommandWaitTimeoutPrintsAcceptedIDs(t *testing.T) { + client := &commandFakeClient{handler: func(path string, _ any, out any) error { + response := `{"ret":"0","data":{"thread":{"run_list":[{"run_id":"run-1","state":1}]}}}` + if path == "/api/biz/v1/skill/canvas/create" { + response = `{"ret":"0","data":{"state":"creating","project_id":"100","thread_id":"thread-1","run_id":"run-1","canvas_asset_id":"200","web_url":"https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200"}}` + } + return json.Unmarshal([]byte(response), out) + }} + var stdout, stderr bytes.Buffer + cmd := NewCommand(&stdout, &stderr, &common.Runner{Client: client}) + cmd.SetArgs([]string{ + "create", "--request-id", "request-1", "--wait", + "--poll-interval", "50ms", "--timeout", "1ms", + }) + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v, want accepted create outcome", err) + } + var result map[string]any + if err := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &result); err != nil { + t.Fatalf("stdout = %q, want JSON: %v", stdout.String(), err) + } + if result["project_id"] != "100" || result["state"] != "creating" || result["warning"] == "" { + t.Fatalf("result = %#v, want accepted IDs and wait warning", result) + } +} + +func TestApplyRequestRejectsTeamOrImportExtensions(t *testing.T) { + for _, field := range []string{"team_id", "provider", "import_source"} { + payload := `{"batch_id":"b","client_id":"c","transactions":[],"` + field + `":"x"}` + _, err := readApplyRequest(strings.NewReader(payload), "-") + if err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("readApplyRequest(%s) error = %v, want unknown field rejection", field, err) + } + } +} diff --git a/cmd/root.go b/cmd/root.go index 1f2598c..1c4f654 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -7,6 +7,7 @@ import ( "strings" // authcmd "github.com/Pippit-dev/pippit-cli/cmd/auth" + canvascmd "github.com/Pippit-dev/pippit-cli/cmd/canvas" "github.com/Pippit-dev/pippit-cli/cmd/generate_image" "github.com/Pippit-dev/pippit-cli/cmd/generate_video" "github.com/Pippit-dev/pippit-cli/cmd/short_drama" @@ -25,7 +26,12 @@ func Execute() error { func NewRootCommand(stdout, stderr io.Writer) *cobra.Command { cfg := config.Load() - client := common.NewHTTPClient(cfg.BaseURL, cfg.HTTPTimeout, common.NewAccessKeyAuthorizer(cfg.AccessKey)) + client := common.NewHTTPClientWithPPEEnv( + cfg.BaseURL, + cfg.HTTPTimeout, + common.NewAccessKeyAuthorizer(cfg.AccessKey), + func() string { return cfg.PPEEnv }, + ) runner := common.NewRunner(cfg, client) return newRootCommand(stdout, stderr, runner) } @@ -43,7 +49,9 @@ func newRootCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Comm root.SetVersionTemplate("{{.Version}}\n") root.SetOut(stdout) root.SetErr(stderr) + configurePPEFlag(root, runner.Config) // root.AddCommand(authcmd.NewCommand(stdout, stderr, runner)) // temporarily disabled; auth is via access key injection + root.AddCommand(canvascmd.NewCommand(stdout, stderr, runner)) root.AddCommand(newDownloadResultCommand(stdout, stderr, runner)) root.AddCommand(newGetThreadCommand(stdout, stderr, runner)) root.AddCommand(newListThreadFileCommand(stdout, stderr, runner)) @@ -58,6 +66,23 @@ func newRootCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Comm return root } +func configurePPEFlag(root *cobra.Command, cfg *config.Config) { + root.PersistentFlags().StringVar( + &cfg.PPEEnv, + "ppe-env", + cfg.PPEEnv, + "route Pippit API requests to a PPE environment (for example, ppe_cli_canvas_ak)", + ) + root.PersistentPreRunE = func(_ *cobra.Command, _ []string) error { + ppeEnv, err := config.NormalizePPEEnv(cfg.PPEEnv) + if err != nil { + return err + } + cfg.PPEEnv = ppeEnv + return nil + } +} + func localizeFlagErrors(cmd *cobra.Command) { cmd.SetFlagErrorFunc(func(_ *cobra.Command, err error) error { return localizeFlagError(err) diff --git a/cmd/root_test.go b/cmd/root_test.go new file mode 100644 index 0000000..daf6c04 --- /dev/null +++ b/cmd/root_test.go @@ -0,0 +1,100 @@ +package cmd + +import ( + "bytes" + "strings" + "testing" + + "github.com/Pippit-dev/pippit-cli/internal/common" + "github.com/Pippit-dev/pippit-cli/internal/config" + "github.com/spf13/cobra" +) + +func TestPPEEnvFlagOverridesEnvironment(t *testing.T) { + t.Setenv(config.EnvPPEEnv, "ppe_from_env") + cfg, root, ran := newPPEFlagTestRoot(t) + root.SetArgs([]string{"ppe-probe", "--ppe-env", "ppe_from_flag"}) + + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if !*ran { + t.Fatal("probe command did not run") + } + if cfg.PPEEnv != "ppe_from_flag" { + t.Fatalf("PPEEnv = %q, want flag value", cfg.PPEEnv) + } +} + +func TestPPEEnvUsesEnvironmentByDefault(t *testing.T) { + t.Setenv(config.EnvPPEEnv, " ppe_from_env ") + cfg, root, ran := newPPEFlagTestRoot(t) + root.SetArgs([]string{"ppe-probe"}) + + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if !*ran { + t.Fatal("probe command did not run") + } + if cfg.PPEEnv != "ppe_from_env" { + t.Fatalf("PPEEnv = %q, want environment value", cfg.PPEEnv) + } +} + +func TestPPEEnvCanBeExplicitlyDisabledByFlag(t *testing.T) { + t.Setenv(config.EnvPPEEnv, "ppe_from_env") + cfg, root, ran := newPPEFlagTestRoot(t) + root.SetArgs([]string{"ppe-probe", "--ppe-env", ""}) + + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if !*ran { + t.Fatal("probe command did not run") + } + if cfg.PPEEnv != "" { + t.Fatalf("PPEEnv = %q, want production", cfg.PPEEnv) + } +} + +func TestPPEEnvRejectsInvalidValueBeforeCommand(t *testing.T) { + t.Setenv(config.EnvPPEEnv, "production") + _, root, ran := newPPEFlagTestRoot(t) + root.SetArgs([]string{"ppe-probe"}) + + err := root.Execute() + if err == nil || !strings.Contains(err.Error(), "PPE 环境") { + t.Fatalf("Execute() error = %v, want invalid PPE error", err) + } + if *ran { + t.Fatal("probe command ran with invalid PPE environment") + } +} + +func TestRootHelpIncludesPPEFlag(t *testing.T) { + var stdout, stderr bytes.Buffer + root := NewRootCommand(&stdout, &stderr) + root.SetArgs([]string{"--help"}) + if err := root.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if !strings.Contains(stdout.String(), "--ppe-env") { + t.Fatalf("help does not include --ppe-env:\n%s", stdout.String()) + } +} + +func newPPEFlagTestRoot(t *testing.T) (*config.Config, *cobra.Command, *bool) { + t.Helper() + cfg := config.Load() + runner := common.NewRunner(cfg, nil) + root := newRootCommand(&bytes.Buffer{}, &bytes.Buffer{}, runner) + ran := false + root.AddCommand(&cobra.Command{ + Use: "ppe-probe", + Run: func(_ *cobra.Command, _ []string) { + ran = true + }, + }) + return cfg, root, &ran +} diff --git a/cmd/short_drama_test.go b/cmd/short_drama_test.go index 53c4f1b..b6706a0 100644 --- a/cmd/short_drama_test.go +++ b/cmd/short_drama_test.go @@ -140,7 +140,7 @@ func TestRootHelpListsSupportedCommands(t *testing.T) { t.Fatalf("help output = %q, want %q", got, want) } } - for _, unwanted := range []string{"completion", "version "} { + for _, unwanted := range []string{"completion", "\n version "} { if strings.Contains(got, unwanted) { t.Fatalf("help output = %q, should not contain %q", got, unwanted) } From 46257b279c09ae7e9aadc22fc94029bb76501deb Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" Date: Tue, 11 Aug 2026 15:18:00 +0800 Subject: [PATCH 05/48] feat: add local LibTV canvas planner Co-authored-by: Codex --- README.md | 36 ++ adapters/libtv/README.md | 50 +++ adapters/libtv/cli.mjs | 82 +++++ adapters/libtv/plan.mjs | 343 ++++++++++++++++++++ adapters/libtv/plan.test.mjs | 102 ++++++ adapters/libtv/testdata/media-manifest.json | 21 ++ adapters/libtv/testdata/snapshot.json | 100 ++++++ package-lock.json | 4 +- package.json | 5 +- scripts/run.js | 15 + scripts/run.test.js | 35 ++ 11 files changed, 789 insertions(+), 4 deletions(-) create mode 100644 adapters/libtv/README.md create mode 100755 adapters/libtv/cli.mjs create mode 100644 adapters/libtv/plan.mjs create mode 100644 adapters/libtv/plan.test.mjs create mode 100644 adapters/libtv/testdata/media-manifest.json create mode 100644 adapters/libtv/testdata/snapshot.json create mode 100644 scripts/run.test.js diff --git a/README.md b/README.md index 6dad317..142667f 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,42 @@ python3 skills/xyq-nest-skill/scripts/download_results.py \ - 超时：连续轮询 48 小时无结果则停止。 - 错误重试：单次失败可重试 1 次，连续 3 次失败则停止。 +## Canvas beta + +Canvas beta 暴露个人漫剧画布的通用原子命令，不在服务端增加任何第三方“导入”语义： + +```bash +npx @pippit-dev/cli@beta install +export XYQ_ACCESS_KEY="" + +# 仅测试 PPE 时配置；生产环境不要设置 +export PIPPIT_CLI_PPE_ENV="ppe_cli_canvas_ak" +# 也可以在任意命令上使用：--ppe-env ppe_cli_canvas_ak + +pippit-tool-cli canvas create --title "Imported draft" --request-id request_001 --wait +pippit-tool-cli canvas upload --path ./clip.mp4 +pippit-tool-cli canvas get --asset-id CANVAS_ASSET_ID +pippit-tool-cli canvas apply --project-id PROJECT_ID --file ./patch.json +``` + +`canvas create/get/apply/upload` 的输出均为单行 JSON，所有资源 ID 保持字符串，便于脚本和 Agent 调用。`create` 的 `request_id` 当前用于追踪和恢复，不是跨服务崩溃窗口的严格幂等键；请求结果不明确时不要盲目重试。beta 的 `apply` 每次只接受一个 transaction（可以包含多个 patches），并严格检查该 transaction 的 ACK 和每个资产版本；当前服务端仍不保证跨资产 all-or-nothing，调用方应在写后执行 `get` 校验，并持久化自己的 operation journal。 + +PPE 只影响 Pippit API 同源请求。CLI 不会把 Access Key、`x-tt-env`、`x-use-ppe` 或 `x-schedule-vdc` 转发给第三方绝对 URL；`--ppe-env` 的优先级高于 `PIPPIT_CLI_PPE_ENV`，二者都未提供时访问生产环境。 + +### LibTV 本地 adapter + +beta 同时提供一个无网络的 LibTV provider adapter，将导出的 snapshot 转成 ID-neutral `pippit-canvas-plan/0.1`： + +```bash +pippit-tool-cli libtv plan \ + --snapshot ./libtv-snapshot.json \ + --media-manifest ./bundle-media.json \ + --title "Imported draft" \ + --output ./canvas-plan.json +``` + +adapter 不读取 Pippit AK、不访问 LibTV、不分配 Pippit ID，也不执行 create/apply。后续 executor 只需把 plan 编译为上述通用 Canvas 命令即可。若 snapshot 只提供带签名参数的素材 URL，plan 会以 `0600` 权限保留它们以供后续下载；不要把 plan 打进日志、提交到仓库或分享给他人。完整边界见 `adapters/libtv/README.md`。 + ## 短剧工作流技能 包发布后可以通过 npm 安装。安装器会按当前系统下载匹配的预构建二进制文件，支持 macOS、Linux 和 Windows： diff --git a/adapters/libtv/README.md b/adapters/libtv/README.md new file mode 100644 index 0000000..8b6f323 --- /dev/null +++ b/adapters/libtv/README.md @@ -0,0 +1,50 @@ +# LibTV canvas adapter + +This directory is a provider adapter, not a Pippit API client. It converts a +LibTV snapshot into the ID-neutral `pippit-canvas-plan/0.1` contract. It does +not read an access key, choose a Pippit environment, allocate Pippit asset IDs, +upload files, create a project, write assets, bind a canvas, or use team state. + +Generate a plan: + +```bash +node adapters/libtv/cli.mjs plan \ + --snapshot ./libtv-snapshot.json \ + --media-manifest ./bundle-media.json \ + --title "My imported canvas" \ + --output ./canvas-plan.json +``` + +`--media-manifest` is optional. It may provide `sourceNodeId` + `fileName` rows +for a local export bundle. Existing prototype manifests may also contain +Pippit IDs or authorization metadata; the adapter deliberately ignores those +fields and never copies them into the plan. If no manifest row exists, the +adapter uses the snapshot's HTTPS media reference and derives a file name. + +The generated plan is written with mode `0600`. When the source export only +contains signed HTTPS media URLs, those URLs (including their query strings) +must remain in the local plan so a later executor can download the files. Treat +the plan as sensitive local state: do not print it into logs, commit it, or +share it. Prefer a local export bundle plus `--media-manifest` when available. + +The generic canvas executor owns the remaining steps: + +1. resolve/download and upload each `required_media` item; +2. create a personal novel canvas; +3. allocate Pippit IDs and materialize the logical nodes/edges/groups; +4. apply a provider-neutral `canvas.write` transaction; +5. query the assets back and verify them. + +Plan IDs (`node:*`, `group:*`, `edge:*`, and `media:*`) are logical and stable +within the source snapshot. They are never Pippit asset IDs. The executor must +allocate new personal asset IDs and keep the logical-to-Pippit mapping in its +resume journal. A plan deliberately has no creation timestamp, so the same +snapshot and media mapping produce byte-for-byte stable JSON; the executor +should hash the complete plan when deriving its operation identity. + +The v0.1 adapter fails closed for unsupported node types or dangling edges. +Supported source types are `group`, `video`, `audio`, and `video-clip`. + +LibTV `video-clip` nodes do not carry a portable generated result. The plan +preserves their input references and records an explicit degradation to an +empty Pippit `video-composite` placeholder. diff --git a/adapters/libtv/cli.mjs b/adapters/libtv/cli.mjs new file mode 100755 index 0000000..44c7997 --- /dev/null +++ b/adapters/libtv/cli.mjs @@ -0,0 +1,82 @@ +#!/usr/bin/env node + +import { chmod, readFile, writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { convertSnapshotToCanvasPlan } from './plan.mjs'; + +function parseArgs(argv) { + const [command, ...rest] = argv; + const args = { command }; + for (let index = 0; index < rest.length; index += 1) { + const key = rest[index]; + const value = rest[index + 1]; + if (!key?.startsWith('--') || !value || value.startsWith('--')) { + throw new Error(`invalid argument near ${key ?? ''}`); + } + args[key.slice(2)] = value; + index += 1; + } + return args; +} + +function required(args, key) { + const value = args[key]?.trim(); + if (!value) throw new Error(`--${key} is required`); + return value; +} + +async function readJson(path) { + const text = await readFile(resolve(path), 'utf8'); + try { + return JSON.parse(text); + } catch (error) { + throw new Error(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`); + } +} + +async function runPlan(args) { + const snapshot = await readJson(required(args, 'snapshot')); + const mediaManifest = args['media-manifest'] ? await readJson(args['media-manifest']) : undefined; + const plan = convertSnapshotToCanvasPlan(snapshot, { mediaManifest, title: args.title }); + const serialized = `${JSON.stringify(plan, null, 2)}\n`; + const output = required(args, 'output'); + if (output === '-') { + process.stdout.write(serialized); + return; + } + const outputPath = resolve(output); + await writeFile(outputPath, serialized, { mode: 0o600 }); + await chmod(outputPath, 0o600); + process.stdout.write(`${JSON.stringify({ + output: outputPath, + schema: plan.schema, + source: plan.source, + media_count: plan.required_media.length, + node_count: plan.nodes.length, + group_count: plan.groups.length, + edge_count: plan.edges.length, + degradation_count: plan.degradations.length, + })}\n`); +} + +async function main(argv = process.argv.slice(2)) { + const args = parseArgs(argv); + if (args.command !== 'plan') { + throw new Error( + 'usage: node adapters/libtv/cli.mjs plan --snapshot ' + + '[--media-manifest ] [--title ] --output <plan.json|->', + ); + } + await runPlan(args); +} + +export { main, parseArgs }; + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + }); +} diff --git a/adapters/libtv/plan.mjs b/adapters/libtv/plan.mjs new file mode 100644 index 0000000..6c65140 --- /dev/null +++ b/adapters/libtv/plan.mjs @@ -0,0 +1,343 @@ +import { createHash } from 'node:crypto'; + +const PLAN_SCHEMA = 'pippit-canvas-plan/0.1'; +const SNAPSHOT_SCHEMA = 'xyq-libtv-snapshot/0.1'; +const SUPPORTED_NODE_TYPES = new Set(['group', 'video', 'audio', 'video-clip']); + +function canonicalize(value) { + if (Array.isArray(value)) return value.map(canonicalize); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, canonicalize(value[key])]), + ); +} + +function sha256Json(value) { + return createHash('sha256').update(JSON.stringify(canonicalize(value))).digest('hex'); +} + +function nonEmptyString(value) { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +} + +function finiteNumber(value, field) { + const number = Number(value); + if (!Number.isFinite(number)) throw new Error(`${field} must be a finite number`); + return number; +} + +function positiveNumber(value, field) { + const number = finiteNumber(value, field); + if (number <= 0) throw new Error(`${field} must be greater than zero`); + return number; +} + +function compact(value) { + return Object.fromEntries(Object.entries(value).filter(([, child]) => child !== undefined)); +} + +function detailDataByNodeId(snapshot) { + const result = new Map(); + for (const item of snapshot.nodeDetails ?? []) { + const sourceNodeId = nonEmptyString(item?.sourceNodeId); + if (!sourceNodeId || result.has(sourceNodeId)) continue; + const data = item?.detail?.data; + result.set(sourceNodeId, data && typeof data === 'object' && !Array.isArray(data) ? data : {}); + } + return result; +} + +function mediaManifestByNodeId(mediaManifest) { + const result = new Map(); + for (const item of mediaManifest?.uploads ?? mediaManifest?.media ?? []) { + const sourceNodeId = nonEmptyString(item?.sourceNodeId ?? item?.source_node_id); + if (!sourceNodeId || result.has(sourceNodeId)) continue; + result.set(sourceNodeId, { + fileName: nonEmptyString(item?.fileName ?? item?.file_name ?? item?.path), + url: normalizeHTTPSURL(item?.url), + }); + } + return result; +} + +function assetReferenceByNodeId(snapshot) { + const result = new Map(); + for (const item of snapshot.assetReferences ?? []) { + const sourceNodeId = nonEmptyString(item?.sourceNodeId); + const url = normalizeHTTPSURL(item?.url); + if (sourceNodeId && url && !result.has(sourceNodeId)) result.set(sourceNodeId, url); + } + return result; +} + +function normalizeHTTPSURL(value) { + const raw = Array.isArray(value) ? value.find((item) => nonEmptyString(item)) : value; + const text = nonEmptyString(raw); + if (!text) return undefined; + let url; + try { + url = new URL(text); + } catch { + throw new Error(`invalid media URL: ${text.slice(0, 120)}`); + } + if (url.protocol !== 'https:') throw new Error(`media URL must use HTTPS: ${url.protocol}`); + return url.toString(); +} + +function fileNameFromURL(value) { + if (!value) return undefined; + const pathname = new URL(value).pathname; + const encodedName = pathname.slice(pathname.lastIndexOf('/') + 1); + if (!encodedName) return undefined; + try { + return decodeURIComponent(encodedName); + } catch { + return encodedName; + } +} + +function safeFileName(value) { + const text = nonEmptyString(value); + if (!text) return undefined; + const normalized = text.replaceAll('\\', '/'); + return normalized.slice(normalized.lastIndexOf('/') + 1) || undefined; +} + +function mediaMetadata(data) { + const item = data?.resourceMeta?.items?.[0] ?? {}; + const duration = Number(item.durationSec); + return compact({ + byte_size: Number(item.byteSize) > 0 ? Number(item.byteSize) : undefined, + duration_ms: Number.isFinite(duration) && duration > 0 ? Math.round(duration * 1000) : undefined, + extension: nonEmptyString(item.extension)?.toLowerCase(), + height: Number(item.height) > 0 ? Number(item.height) : undefined, + mime_type: nonEmptyString(item.mimeType), + width: Number(item.width) > 0 ? Number(item.width) : undefined, + }); +} + +function fallbackFileName(node, metadata) { + const extension = metadata.extension ?? (node.type === 'audio' ? 'audio' : 'video'); + const stem = (nonEmptyString(node.name) ?? node.id) + .replaceAll(/[\\/:*?"<>|]/g, '_') + .slice(0, 120); + return `${stem}.${extension}`; +} + +function logicalNodeId(sourceNodeId) { + return `node:${sourceNodeId}`; +} + +function logicalGroupId(sourceNodeId) { + return `group:${sourceNodeId}`; +} + +function logicalMediaId(sourceNodeId) { + return `media:${sourceNodeId}`; +} + +function logicalEdgeId(sourceEdgeId) { + return `edge:${sourceEdgeId}`; +} + +function validateSnapshot(snapshot) { + if (snapshot?.protocolVersion !== SNAPSHOT_SCHEMA) { + throw new Error(`unsupported LibTV snapshot schema: ${snapshot?.protocolVersion ?? '<missing>'}`); + } + const projectId = nonEmptyString(snapshot?.project?.projectUuid); + if (!projectId) throw new Error('snapshot project.projectUuid is required'); + const nodes = snapshot?.project?.nodes; + if (!Array.isArray(nodes) || nodes.length === 0) throw new Error('snapshot project.nodes must not be empty'); + if (!Array.isArray(snapshot?.project?.edges)) throw new Error('snapshot project.edges must be an array'); + + const nodeById = new Map(); + nodes.forEach((node, index) => { + const id = nonEmptyString(node?.id); + if (!id) throw new Error(`snapshot node ${index} has no id`); + if (nodeById.has(id)) throw new Error(`duplicate snapshot node id: ${id}`); + if (!SUPPORTED_NODE_TYPES.has(node?.type)) throw new Error(`unsupported LibTV node type: ${node?.type ?? '<missing>'}`); + finiteNumber(node?.position?.x, `node ${id} position.x`); + finiteNumber(node?.position?.y, `node ${id} position.y`); + positiveNumber(node?.width, `node ${id} width`); + positiveNumber(node?.height, `node ${id} height`); + nodeById.set(id, node); + }); + + for (const node of nodes) { + const parentId = nonEmptyString(node?.parentId); + if (parentId && nodeById.get(parentId)?.type !== 'group') { + throw new Error(`node ${node.id} references a missing or non-group parent ${parentId}`); + } + } + + const edgeIds = new Set(); + snapshot.project.edges.forEach((edge, index) => { + const id = nonEmptyString(edge?.id); + if (!id) throw new Error(`snapshot edge ${index} has no id`); + if (edgeIds.has(id)) throw new Error(`duplicate snapshot edge id: ${id}`); + edgeIds.add(id); + if (!nodeById.has(edge?.source) || !nodeById.has(edge?.target)) { + throw new Error(`edge ${id} references a missing node`); + } + if (nodeById.get(edge.source).type === 'group' || nodeById.get(edge.target).type === 'group') { + throw new Error(`edge ${id} must connect business nodes, not groups`); + } + }); + return { projectId, nodes, nodeById }; +} + +function videoCompositeInputs(data, nodeById) { + const preferred = Array.isArray(data?.params?.videoList) + ? data.params.videoList.map((item) => item?.nodeId) + : []; + const fallback = Array.isArray(data?.clipTimelineData?.videoSourceNodeIds) + ? data.clipTimelineData.videoSourceNodeIds + : []; + const chosen = preferred.some(Boolean) ? preferred : fallback; + const unique = []; + const seen = new Set(); + for (const sourceId of chosen) { + if (!nonEmptyString(sourceId) || seen.has(sourceId) || nodeById.get(sourceId)?.type !== 'video') continue; + seen.add(sourceId); + unique.push(logicalNodeId(sourceId)); + } + return unique; +} + +function sourceFingerprint(snapshot) { + return `sha256:${sha256Json({ + protocolVersion: snapshot.protocolVersion, + project: snapshot.project, + nodeDetails: snapshot.nodeDetails ?? [], + assetReferences: snapshot.assetReferences ?? [], + })}`; +} + +function canvasTitle(snapshot, projectId, override) { + const title = nonEmptyString(override) ?? nonEmptyString(snapshot.project.name) ?? `LibTV · ${projectId}`; + if (Array.from(title).length > 50) { + throw new Error('canvas title must not exceed 50 characters; provide a shorter --title'); + } + return title; +} + +function convertSnapshotToCanvasPlan(snapshot, options = {}) { + const { projectId, nodes: sourceNodes, nodeById } = validateSnapshot(snapshot); + const details = detailDataByNodeId(snapshot); + const manifest = mediaManifestByNodeId(options.mediaManifest); + const assetReferences = assetReferenceByNodeId(snapshot); + const requiredMedia = []; + const nodes = []; + const groups = []; + const degradations = []; + + sourceNodes.forEach((sourceNode, order) => { + const sourceNodeId = sourceNode.id; + const position = { + x: finiteNumber(sourceNode.position.x, `node ${sourceNodeId} position.x`), + y: finiteNumber(sourceNode.position.y, `node ${sourceNodeId} position.y`), + }; + const size = { + width: positiveNumber(sourceNode.width, `node ${sourceNodeId} width`), + height: positiveNumber(sourceNode.height, `node ${sourceNodeId} height`), + }; + if (sourceNode.type === 'group') { + const children = sourceNodes + .filter((child) => child.parentId === sourceNodeId) + .map((child) => child.type === 'group' ? logicalGroupId(child.id) : logicalNodeId(child.id)); + groups.push({ + logical_id: logicalGroupId(sourceNodeId), + source_node_id: sourceNodeId, + title: nonEmptyString(sourceNode.name) ?? 'LibTV group', + position, + size, + order, + child_logical_ids: children, + }); + return; + } + + const detail = details.get(sourceNodeId) ?? {}; + const base = compact({ + logical_id: logicalNodeId(sourceNodeId), + source_node_id: sourceNodeId, + title: nonEmptyString(sourceNode.name) ?? sourceNodeId, + position, + size, + parent_group_logical_id: sourceNode.parentId ? logicalGroupId(sourceNode.parentId) : undefined, + order, + }); + if (sourceNode.type === 'video-clip') { + const inputNodeLogicalIds = videoCompositeInputs(detail, nodeById); + nodes.push({ + ...base, + kind: 'video-composite', + target_type: 'biz/video', + variant: 'video-composite', + input_node_logical_ids: inputNodeLogicalIds, + }); + degradations.push({ + code: 'libtv.video_clip.empty_placeholder', + source_node_id: sourceNodeId, + message: 'LibTV video-clip has no portable generated result and will become an empty video-composite placeholder.', + input_node_logical_ids: inputNodeLogicalIds, + }); + return; + } + + const mediaType = sourceNode.type; + const manifestItem = manifest.get(sourceNodeId) ?? {}; + const url = manifestItem.url ?? assetReferences.get(sourceNodeId) ?? normalizeHTTPSURL(detail?.url); + const metadata = mediaMetadata(detail); + const fileName = safeFileName(manifestItem.fileName) ?? safeFileName(fileNameFromURL(url)) ?? fallbackFileName(sourceNode, metadata); + const mediaLogicalId = logicalMediaId(sourceNodeId); + requiredMedia.push(compact({ + logical_id: mediaLogicalId, + source_node_id: sourceNodeId, + file_name: fileName, + media_type: mediaType, + url, + metadata, + })); + nodes.push({ + ...base, + kind: mediaType, + target_type: mediaType === 'audio' ? 'biz/audio' : 'biz/video', + media_logical_id: mediaLogicalId, + }); + }); + + const edges = snapshot.project.edges.map((edge) => ({ + logical_id: logicalEdgeId(edge.id), + source_edge_id: edge.id, + type: 'reference', + source_node_logical_id: logicalNodeId(edge.source), + target_node_logical_id: logicalNodeId(edge.target), + source_handle: 'right', + target_handle: 'left', + })); + + return { + schema: PLAN_SCHEMA, + title: canvasTitle(snapshot, projectId, options.title), + source: { + provider: 'libtv', + project_id: projectId, + fingerprint: sourceFingerprint(snapshot), + }, + required_media: requiredMedia, + nodes, + groups, + edges, + degradations, + }; +} + +export { + PLAN_SCHEMA, + SNAPSHOT_SCHEMA, + convertSnapshotToCanvasPlan, + sha256Json, +}; diff --git a/adapters/libtv/plan.test.mjs b/adapters/libtv/plan.test.mjs new file mode 100644 index 0000000..5b4e780 --- /dev/null +++ b/adapters/libtv/plan.test.mjs @@ -0,0 +1,102 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; + +import { PLAN_SCHEMA, convertSnapshotToCanvasPlan } from './plan.mjs'; + +const fixtureURL = new URL('./testdata/snapshot.json', import.meta.url); +const manifestURL = new URL('./testdata/media-manifest.json', import.meta.url); + +async function readJson(url) { + return JSON.parse(await readFile(url, 'utf8')); +} + +function clone(value) { + return JSON.parse(JSON.stringify(value)); +} + +async function testRepresentativeSnapshot() { + const snapshot = await readJson(fixtureURL); + const mediaManifest = await readJson(manifestURL); + const plan = convertSnapshotToCanvasPlan(snapshot, { mediaManifest }); + + assert.equal(plan.schema, PLAN_SCHEMA); + assert.deepEqual(plan.source, { + provider: 'libtv', + project_id: 'fixture-project', + fingerprint: 'sha256:c33d2d9580d2a3a57fd06cac45e44f8881cdd70df6aeffc178e0f6077c11cf68', + }); + assert.equal(plan.title, 'LibTV adapter fixture'); + assert.equal(plan.required_media.length, 2); + assert.deepEqual(plan.required_media[0], { + logical_id: 'media:video-1', + source_node_id: 'video-1', + file_name: 'input.mp4', + media_type: 'video', + url: 'https://media.example.test/input.mp4?token=fixture', + metadata: { + byte_size: 1234, + duration_ms: 2020, + extension: 'mp4', + height: 180, + mime_type: 'video/mp4', + width: 320, + }, + }); + assert.equal(plan.required_media[1].file_name, 'input.wav'); + assert.equal(plan.required_media[1].url, undefined); + assert.equal(plan.nodes.length, 3); + assert.deepEqual(plan.nodes[2].input_node_logical_ids, ['node:video-1']); + assert.deepEqual(plan.groups[0].child_logical_ids, ['node:video-1', 'node:audio-1']); + assert.deepEqual(plan.edges[0], { + logical_id: 'edge:edge-1', + source_edge_id: 'edge-1', + type: 'reference', + source_node_logical_id: 'node:video-1', + target_node_logical_id: 'node:clip-1', + source_handle: 'right', + target_handle: 'left', + }); + assert.equal(plan.degradations.length, 1); + assert.equal(plan.degradations[0].code, 'libtv.video_clip.empty_placeholder'); + + const serialized = JSON.stringify(plan); + for (const forbidden of ['must-not-leak', 'assetId', 'pippitAssetId', 'teamId', 'access_key']) { + assert.equal(serialized.includes(forbidden), false, `plan leaked forbidden field/value ${forbidden}`); + } +} + +async function testDeterminismAndVolatileExportTime() { + const snapshot = await readJson(fixtureURL); + const first = convertSnapshotToCanvasPlan(snapshot); + const second = convertSnapshotToCanvasPlan({ ...snapshot, exportedAt: '2099-01-01T00:00:00.000Z' }); + assert.deepEqual(first, second); + assert.equal(convertSnapshotToCanvasPlan(snapshot, { title: 'Override title' }).title, 'Override title'); +} + +async function testRejectsUnsafeOrInvalidInputs() { + const snapshot = await readJson(fixtureURL); + const unsafe = clone(snapshot); + unsafe.nodeDetails[0].detail.data.url = ['http://media.example.test/input.mp4']; + unsafe.assetReferences = []; + assert.throws(() => convertSnapshotToCanvasPlan(unsafe), /must use HTTPS/); + + const dangling = clone(snapshot); + dangling.project.edges[0].target = 'missing-node'; + assert.throws(() => convertSnapshotToCanvasPlan(dangling), /references a missing node/); + + const unsupported = clone(snapshot); + unsupported.project.nodes[1].type = 'prompt'; + assert.throws(() => convertSnapshotToCanvasPlan(unsupported), /unsupported LibTV node type/); + + assert.throws(() => convertSnapshotToCanvasPlan(snapshot, { title: 'x'.repeat(51) }), /must not exceed 50/); + + const encodedSlash = clone(snapshot); + encodedSlash.assetReferences[0].url = 'https://media.example.test/nested%2Fevil.mp4'; + encodedSlash.nodeDetails[0].detail.data.url = []; + assert.equal(convertSnapshotToCanvasPlan(encodedSlash).required_media[0].file_name, 'evil.mp4'); +} + +await testRepresentativeSnapshot(); +await testDeterminismAndVolatileExportTime(); +await testRejectsUnsafeOrInvalidInputs(); +process.stdout.write('libtv plan adapter tests passed\n'); diff --git a/adapters/libtv/testdata/media-manifest.json b/adapters/libtv/testdata/media-manifest.json new file mode 100644 index 0000000..32d5394 --- /dev/null +++ b/adapters/libtv/testdata/media-manifest.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": "pippit-canvas-import-upload-manifest/0.1", + "auth": { + "mode": "access_key", + "teamId": "must-not-leak" + }, + "uploads": [ + { + "sourceNodeId": "video-1", + "fileName": "nested/input.mp4", + "assetId": "must-not-leak", + "pippitAssetId": "must-not-leak" + }, + { + "sourceNodeId": "audio-1", + "fileName": "input.wav", + "assetId": "must-not-leak", + "pippitAssetId": "must-not-leak" + } + ] +} diff --git a/adapters/libtv/testdata/snapshot.json b/adapters/libtv/testdata/snapshot.json new file mode 100644 index 0000000..46f2be1 --- /dev/null +++ b/adapters/libtv/testdata/snapshot.json @@ -0,0 +1,100 @@ +{ + "protocolVersion": "xyq-libtv-snapshot/0.1", + "exportedAt": "2026-08-10T06:46:08.883Z", + "source": { + "platform": "libtv", + "projectId": "fixture-project" + }, + "project": { + "projectUuid": "fixture-project", + "name": "LibTV adapter fixture", + "nodes": [ + { + "id": "group-1", + "name": "Media group", + "type": "group", + "position": { "x": 10, "y": 20 }, + "width": 1000, + "height": 500 + }, + { + "id": "video-1", + "name": "Input video", + "type": "video", + "position": { "x": 30, "y": 40 }, + "parentId": "group-1", + "width": 622, + "height": 350 + }, + { + "id": "audio-1", + "name": "Input audio", + "type": "audio", + "position": { "x": 30, "y": 410 }, + "parentId": "group-1", + "width": 350, + "height": 148 + }, + { + "id": "clip-1", + "name": "Smart edit", + "type": "video-clip", + "position": { "x": 1100, "y": 20 }, + "width": 350, + "height": 350 + } + ], + "edges": [ + { "id": "edge-1", "source": "video-1", "target": "clip-1" } + ] + }, + "nodeDetails": [ + { + "sourceNodeId": "video-1", + "detail": { + "data": { + "type": "video", + "url": ["https://media.example.test/input.mp4?token=fixture"], + "resourceMeta": { + "items": [ + { + "byteSize": 1234, + "durationSec": 2.02, + "extension": "mp4", + "height": 180, + "mimeType": "video/mp4", + "width": 320 + } + ] + } + } + } + }, + { + "sourceNodeId": "audio-1", + "detail": { + "data": { + "type": "audio", + "resourceMeta": { "items": [{ "durationSec": 2, "extension": "wav" }] } + } + } + }, + { + "sourceNodeId": "clip-1", + "detail": { + "data": { + "type": "video-clip", + "params": { "videoList": [{ "nodeId": "video-1" }] }, + "clipTimelineData": { "videoSourceNodeIds": [] } + } + } + } + ], + "assetReferences": [ + { + "sourceNodeId": "video-1", + "field": "url", + "url": "https://media.example.test/input.mp4?token=fixture" + } + ] +} diff --git a/package-lock.json b/package-lock.json index 87e5f06..67ee779 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@pippit-dev/cli", - "version": "1.0.17", + "version": "1.1.0-beta.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@pippit-dev/cli", - "version": "1.0.17", + "version": "1.1.0-beta.1", "hasInstallScript": true, "license": "MIT", "bin": { diff --git a/package.json b/package.json index 7fe7bfc..4ed2119 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,13 @@ { "name": "@pippit-dev/cli", - "version": "1.0.17", + "version": "1.1.0-beta.1", "description": "Pippit CLI", "bin": { "pippit-tool-cli": "scripts/run.js" }, "scripts": { "postinstall": "node scripts/install.js", - "test": "node scripts/version-check.test.js && node scripts/skills.test.js && node scripts/install-wizard.test.js && go test ./... && go vet ./..." + "test": "node scripts/version-check.test.js && node scripts/skills.test.js && node scripts/install-wizard.test.js && node scripts/run.test.js && node adapters/libtv/plan.test.mjs && go test ./... && go vet ./..." }, "os": [ "darwin", @@ -29,6 +29,7 @@ "files": [ "cmd", "internal", + "adapters", "skills", "scripts/install.js", "scripts/install-wizard.js", diff --git a/scripts/run.js b/scripts/run.js index 7e27f85..d569437 100755 --- a/scripts/run.js +++ b/scripts/run.js @@ -39,6 +39,21 @@ if (process.platform === "win32" && fs.existsSync(oldBin)) { // should run the JS setup flow before the native binary exists. if (args[0] === "install") { require("./install-wizard.js").main(); +} else if (args[0] === "libtv") { + // The LibTV adapter is intentionally local-only and runs before the native + // binary is installed. It never receives Pippit credentials or PPE headers. + const adapterEnv = { ...process.env }; + delete adapterEnv.XYQ_ACCESS_KEY; + delete adapterEnv.PIPPIT_ACCESS_KEY; + delete adapterEnv.PIPPIT_CLI_PPE_ENV; + try { + execFileSync(process.execPath, [ + path.join(__dirname, "..", "adapters", "libtv", "cli.mjs"), + ...args.slice(1), + ], { stdio: "inherit", env: adapterEnv }); + } catch (e) { + process.exit(e.status || 1); + } } else { maybeWarnNewVersion(args); diff --git a/scripts/run.test.js b/scripts/run.test.js new file mode 100644 index 0000000..49fa9cc --- /dev/null +++ b/scripts/run.test.js @@ -0,0 +1,35 @@ +const assert = require("assert"); +const { spawnSync } = require("child_process"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const root = path.join(__dirname, ".."); +const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "pippit-libtv-run-")); +const output = path.join(outputDir, "plan.json"); + +assert.deepStrictEqual(require("../package.json").bin, { + "pippit-tool-cli": "scripts/run.js", +}); + +try { + const result = spawnSync(process.execPath, [ + path.join(__dirname, "run.js"), + "libtv", + "plan", + "--snapshot", + path.join(root, "adapters", "libtv", "testdata", "snapshot.json"), + "--output", + output, + ], { + cwd: root, + encoding: "utf8", + env: { ...process.env, PIPPIT_CLI_SKIP_VERSION_CHECK: "1" }, + }); + + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(JSON.parse(fs.readFileSync(output, "utf8")).schema, "pippit-canvas-plan/0.1"); + assert.strictEqual(JSON.parse(result.stdout).schema, "pippit-canvas-plan/0.1"); +} finally { + fs.rmSync(outputDir, { recursive: true, force: true }); +} From 6ea42f515c28df4a0b69f90e8dd100db9be638b8 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 15:18:18 +0800 Subject: [PATCH 06/48] ci: add beta release channel Co-authored-by: Codex <codex@openai.com> --- .github/workflows/release.yml | 29 ++++++++++++++- .github/workflows/test.yml | 27 ++++++++++++++ cmd/update/update.go | 19 +++++----- cmd/update/update_test.go | 19 +++++----- scripts/install-wizard.js | 7 +++- scripts/install-wizard.test.js | 18 ++++++++- scripts/install.js | 10 ++++- scripts/telemetry.js | 2 +- scripts/version-check.js | 60 ++++++++++++++++++++++++------ scripts/version-check.test.js | 68 +++++++++++++++++++++++++++++++++- 10 files changed, 221 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a837a76..a1dad49 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,33 @@ jobs: node-version: "20" registry-url: "https://registry.npmjs.org" + - name: Select npm dist-tag + id: npm_channel + shell: bash + run: | + set -euo pipefail + version="$(node -p "require('./package.json').version")" + if [[ "${GITHUB_REF_NAME}" != "v${version}" ]]; then + echo "Tag ${GITHUB_REF_NAME} does not match package.json version ${version}" >&2 + exit 1 + fi + case "${version}" in + *-beta.*) + dist_tag="beta" + ;; + *-*) + echo "Unsupported prerelease channel in ${version}; only beta releases are publishable" >&2 + exit 1 + ;; + *) + dist_tag="latest" + ;; + esac + echo "dist_tag=${dist_tag}" >> "${GITHUB_OUTPUT}" + + - name: Run tests + run: npm test + - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 with: @@ -40,6 +67,6 @@ jobs: cp dist/checksums.txt checksums.txt - name: Publish to npm - run: npm publish --provenance --access public + run: npm publish --provenance --access public --tag "${{ steps.npm_channel.outputs.dist_tag }}" env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..83ebd75 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,27 @@ +name: Test + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Run tests + run: npm test diff --git a/cmd/update/update.go b/cmd/update/update.go index c149225..9432290 100644 --- a/cmd/update/update.go +++ b/cmd/update/update.go @@ -54,7 +54,7 @@ func NewCommand(stdout, stderr io.Writer) *cobra.Command { func runUpdate(stdout, stderr io.Writer) error { pkg := os.Getenv("PIPPIT_CLI_INSTALL_PACKAGE") if pkg == "" { - pkg = defaultPackage + "@latest" + pkg = defaultInstallPackage(version.Current()) } fmt.Fprintf(stderr, "Updating pippit-tool-cli via npm: %s\n", pkg) @@ -82,6 +82,14 @@ func runUpdate(stdout, stderr io.Writer) error { return nil } +func defaultInstallPackage(currentVersion string) string { + channel := "latest" + if strings.Contains(strings.TrimSpace(currentVersion), "-beta.") { + channel = "beta" + } + return defaultPackage + "@" + channel +} + func globalPackageRoot(pkg string) (string, error) { out, err := command("npm", "root", "-g").Output() if err != nil { @@ -203,14 +211,7 @@ func telemetryBaseURL() string { } func telemetryCliVersion() string { - return stripPrereleaseVersion(version.Current()) -} - -func stripPrereleaseVersion(value string) string { - if idx := strings.Index(value, "-"); idx >= 0 { - return value[:idx] - } - return value + return version.Current() } func runInherit(stderr io.Writer, name string, args ...string) error { diff --git a/cmd/update/update_test.go b/cmd/update/update_test.go index c3c9cfc..41c357c 100644 --- a/cmd/update/update_test.go +++ b/cmd/update/update_test.go @@ -159,15 +159,16 @@ func TestTelemetryBaseURL(t *testing.T) { } } -func TestStripPrereleaseVersion(t *testing.T) { - cases := map[string]string{ - "0.0.27": "0.0.27", - "0.0.27-rc.1": "0.0.27", - "1.2.3-beta.4": "1.2.3", - } - for input, want := range cases { - if got := stripPrereleaseVersion(input); got != want { - t.Fatalf("stripPrereleaseVersion(%q) = %q, want %q", input, got, want) +func TestDefaultInstallPackageFollowsCurrentReleaseChannel(t *testing.T) { + tests := map[string]string{ + "1.2.3": "@pippit-dev/cli@latest", + "1.2.3-beta.4": "@pippit-dev/cli@beta", + " 1.2.3-beta.4 ": "@pippit-dev/cli@beta", + "dev": "@pippit-dev/cli@latest", + } + for currentVersion, want := range tests { + if got := defaultInstallPackage(currentVersion); got != want { + t.Fatalf("defaultInstallPackage(%q) = %q, want %q", currentVersion, got, want) } } } diff --git a/scripts/install-wizard.js b/scripts/install-wizard.js index 62a8aeb..06c5a4e 100755 --- a/scripts/install-wizard.js +++ b/scripts/install-wizard.js @@ -6,7 +6,11 @@ const { isWindows, run, runSilent } = require("./platform"); const { DEFAULT_PKG, installGlobalPackageSkills } = require("./skills"); const { reportBundledSkillTelemetry } = require("./telemetry"); -const VERSION = require("../package.json").version.replace(/-.*$/, ""); +function exactPackageVersion(value) { + return String(value || "").trim(); +} + +const VERSION = exactPackageVersion(require("../package.json").version); function defaultInstallPackage() { return `${DEFAULT_PKG}@${VERSION}`; @@ -88,6 +92,7 @@ if (require.main === module) { module.exports = { defaultInstallPackage, + exactPackageVersion, installPackage, main, }; diff --git a/scripts/install-wizard.test.js b/scripts/install-wizard.test.js index 651da27..f597a89 100644 --- a/scripts/install-wizard.test.js +++ b/scripts/install-wizard.test.js @@ -1,9 +1,18 @@ const assert = require("assert"); -const { defaultInstallPackage, installPackage } = require("./install-wizard"); +const { + defaultInstallPackage, + exactPackageVersion: wizardPackageVersion, + installPackage, +} = require("./install-wizard"); +const { + archiveName, + exactPackageVersion: installerPackageVersion, + releaseURL, +} = require("./install"); const { DEFAULT_PKG } = require("./skills"); -const version = require("../package.json").version.replace(/-.*$/, ""); +const version = require("../package.json").version; delete process.env.PIPPIT_CLI_INSTALL_PACKAGE; assert.strictEqual(defaultInstallPackage(), `${DEFAULT_PKG}@${version}`); @@ -11,3 +20,8 @@ assert.strictEqual(installPackage(), `${DEFAULT_PKG}@${version}`); process.env.PIPPIT_CLI_INSTALL_PACKAGE = `${DEFAULT_PKG}@0.0.26`; assert.strictEqual(installPackage(), `${DEFAULT_PKG}@0.0.26`); + +assert.strictEqual(wizardPackageVersion("1.1.0-beta.3"), "1.1.0-beta.3"); +assert.strictEqual(installerPackageVersion("1.1.0-beta.3"), "1.1.0-beta.3"); +assert.ok(archiveName.includes(`-${version}-`), archiveName); +assert.ok(releaseURL.includes(`/download/v${version}/`), releaseURL); diff --git a/scripts/install.js b/scripts/install.js index d618762..441fc4d 100644 --- a/scripts/install.js +++ b/scripts/install.js @@ -8,7 +8,11 @@ const { isWindows, run } = require("./platform"); const { cleanupLegacyGlobalSkills, installSkillsFromRoot } = require("./skills"); const { reportBundledSkillTelemetry } = require("./telemetry"); -const VERSION = require("../package.json").version.replace(/-.*$/, ""); +function exactPackageVersion(value) { + return String(value || "").trim(); +} + +const VERSION = exactPackageVersion(require("../package.json").version); const REPO = "Pippit-dev/cli"; const NAME = "pippit-tool-cli"; const ROOT = path.join(__dirname, ".."); @@ -159,7 +163,7 @@ if (require.main === module) { console.error(`Failed to install ${NAME}: ${err.message || err}`); console.error( "\nTry:\n" + - " npm install -g @pippit-dev/cli\n" + + ` npm install -g @pippit-dev/cli@${VERSION}\n` + ` node "${path.join(__dirname, "install.js")}"\n` ); process.exit(1); @@ -168,7 +172,9 @@ if (require.main === module) { module.exports = { archiveName, + exactPackageVersion, expectedChecksum, install, + releaseURL, verifyChecksum, }; diff --git a/scripts/telemetry.js b/scripts/telemetry.js index a4fd1d8..3374a7a 100644 --- a/scripts/telemetry.js +++ b/scripts/telemetry.js @@ -2,7 +2,7 @@ const os = require("os"); const http = require("http"); const https = require("https"); -const VERSION = require("../package.json").version.replace(/-.*$/, ""); +const VERSION = require("../package.json").version; const DEFAULT_BASE_URL = "https://xyq.jianying.com"; const REPORT_PATH = "/api/biz/v1/skill/report_telemetry"; const AUTH_HEADER = "Bearer pippit-cli-skill-telemetry"; diff --git a/scripts/version-check.js b/scripts/version-check.js index c2fb42d..a223279 100644 --- a/scripts/version-check.js +++ b/scripts/version-check.js @@ -11,26 +11,56 @@ function defaultCacheFile() { } function currentVersion() { - return require("../package.json").version.replace(/-.*$/, ""); + return require("../package.json").version; } function parseSemver(version) { - const match = String(version || "").trim().match(/^v?(\d+)\.(\d+)\.(\d+)/); + const match = String(version || "").trim().match( + /^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/ + ); if (!match) return null; - return match.slice(1).map(Number); + return { + major: Number(match[1]), + minor: Number(match[2]), + patch: Number(match[3]), + prerelease: match[4] ? match[4].split(".") : [], + }; } function compareSemver(a, b) { const parsedA = parseSemver(a); const parsedB = parseSemver(b); if (!parsedA || !parsedB) return 0; - for (let i = 0; i < 3; i++) { - const diff = parsedA[i] - parsedB[i]; + for (const key of ["major", "minor", "patch"]) { + const diff = parsedA[key] - parsedB[key]; if (diff !== 0) return diff; } + + if (parsedA.prerelease.length === 0 || parsedB.prerelease.length === 0) { + if (parsedA.prerelease.length === parsedB.prerelease.length) return 0; + return parsedA.prerelease.length === 0 ? 1 : -1; + } + const count = Math.max(parsedA.prerelease.length, parsedB.prerelease.length); + for (let i = 0; i < count; i++) { + const identifierA = parsedA.prerelease[i]; + const identifierB = parsedB.prerelease[i]; + if (identifierA === undefined) return -1; + if (identifierB === undefined) return 1; + if (identifierA === identifierB) continue; + const numericA = /^\d+$/.test(identifierA); + const numericB = /^\d+$/.test(identifierB); + if (numericA && numericB) return Number(identifierA) - Number(identifierB); + if (numericA !== numericB) return numericA ? -1 : 1; + return identifierA < identifierB ? -1 : 1; + } return 0; } +function distTagForVersion(version) { + const parsed = parseSemver(version); + return parsed && parsed.prerelease[0] === "beta" ? "beta" : "latest"; +} + function readCache(cacheFile) { try { return JSON.parse(fs.readFileSync(cacheFile, "utf8")); @@ -48,8 +78,8 @@ function writeCache(cacheFile, data) { } } -function fetchLatestVersion(pkg = DEFAULT_PKG) { - return runSilent("npm", ["view", pkg, "version"], { timeout: 3000 }).toString().trim(); +function fetchLatestVersion(pkg = DEFAULT_PKG, distTag = "latest") { + return runSilent("npm", ["view", `${pkg}@${distTag}`, "version"], { timeout: 3000 }).toString().trim(); } function shouldSkip(args, env) { @@ -69,29 +99,35 @@ function maybeWarnNewVersion(args = [], opts = {}) { const now = opts.now || Date.now(); const cacheFile = opts.cacheFile || defaultCacheFile(); const cache = readCache(cacheFile); - const cacheFresh = cache && now - cache.checkedAt < CHECK_INTERVAL_MS; + const current = opts.currentVersion || currentVersion(); + const channel = distTagForVersion(current); + const cacheFresh = cache && cache.channel === channel && now - cache.checkedAt < CHECK_INTERVAL_MS; let latest = cacheFresh ? cache.latest : ""; if (!latest) { try { - latest = (opts.fetchLatestVersion || fetchLatestVersion)(opts.pkg || DEFAULT_PKG); - writeCache(cacheFile, { latest, checkedAt: now }); + latest = (opts.fetchLatestVersion || fetchLatestVersion)(opts.pkg || DEFAULT_PKG, channel); + writeCache(cacheFile, { channel, latest, checkedAt: now }); } catch (_) { return; } } - const current = opts.currentVersion || currentVersion(); if (compareSemver(latest, current) <= 0) return; const warn = opts.warn || console.error; - warn(`[pippit-tool-cli] New version available: ${current} -> ${latest}. Run: pippit-tool-cli update`); + const updateCommand = channel === "beta" + ? `npx ${opts.pkg || DEFAULT_PKG}@beta install` + : "pippit-tool-cli update"; + warn(`[pippit-tool-cli] New version available: ${current} -> ${latest}. Run: ${updateCommand}`); } module.exports = { CHECK_INTERVAL_MS, compareSemver, defaultCacheFile, + distTagForVersion, + fetchLatestVersion, maybeWarnNewVersion, parseSemver, }; diff --git a/scripts/version-check.test.js b/scripts/version-check.test.js index 5404a9b..71e6979 100644 --- a/scripts/version-check.test.js +++ b/scripts/version-check.test.js @@ -1,8 +1,74 @@ const assert = require("assert"); +const fs = require("fs"); +const os = require("os"); const path = require("path"); -const { defaultCacheFile } = require("./version-check"); +const { + compareSemver, + defaultCacheFile, + distTagForVersion, + maybeWarnNewVersion, + parseSemver, +} = require("./version-check"); assert.strictEqual( defaultCacheFile(), path.join(require("os").homedir(), ".pippit_tool_cli", "version-check.json") ); + +assert.deepStrictEqual(parseSemver("v1.2.3-beta.4+build.7"), { + major: 1, + minor: 2, + patch: 3, + prerelease: ["beta", "4"], +}); +assert.strictEqual(compareSemver("1.2.3-beta.2", "1.2.3-beta.1"), 1); +assert.strictEqual(compareSemver("1.2.3-beta.10", "1.2.3-beta.2"), 8); +assert.strictEqual(compareSemver("1.2.3", "1.2.3-beta.99"), 1); +assert.strictEqual(compareSemver("1.2.3-beta.1", "1.2.3"), -1); +assert.strictEqual(distTagForVersion("1.2.3-beta.1"), "beta"); +assert.strictEqual(distTagForVersion("1.2.3"), "latest"); + +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "pippit-version-check-")); +try { + const cacheFile = path.join(tmpDir, "cache.json"); + const betaWarnings = []; + let betaFetches = 0; + maybeWarnNewVersion([], { + cacheFile, + currentVersion: "1.1.0-beta.1", + env: {}, + fetchLatestVersion(pkg, tag) { + betaFetches += 1; + assert.strictEqual(pkg, "@pippit-dev/cli"); + assert.strictEqual(tag, "beta"); + return "1.1.0-beta.2"; + }, + now: 1000, + warn: (message) => betaWarnings.push(message), + }); + assert.strictEqual(betaFetches, 1); + assert.strictEqual(betaWarnings.length, 1); + assert.ok(betaWarnings[0].includes("1.1.0-beta.1 -> 1.1.0-beta.2")); + assert.ok(betaWarnings[0].includes("npx @pippit-dev/cli@beta install")); + assert.deepStrictEqual(JSON.parse(fs.readFileSync(cacheFile, "utf8")), { + channel: "beta", + latest: "1.1.0-beta.2", + checkedAt: 1000, + }); + + let stableFetches = 0; + maybeWarnNewVersion([], { + cacheFile, + currentVersion: "1.0.17", + env: {}, + fetchLatestVersion(_pkg, tag) { + stableFetches += 1; + assert.strictEqual(tag, "latest"); + return "1.0.17"; + }, + now: 1001, + }); + assert.strictEqual(stableFetches, 1, "beta cache must not satisfy latest checks"); +} finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); +} From 463f6adec06be862a63014a33dfdb61d40bc51a2 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:45:57 +0800 Subject: [PATCH 07/48] feat: define provider-neutral CanvasPlan contract Co-authored-by: Codex <codex@openai.com> --- internal/canvasplan/decode.go | 69 ++++++ internal/canvasplan/hash.go | 40 +++ internal/canvasplan/types.go | 199 +++++++++++++++ internal/canvasplan/validate.go | 414 ++++++++++++++++++++++++++++++++ 4 files changed, 722 insertions(+) create mode 100644 internal/canvasplan/decode.go create mode 100644 internal/canvasplan/hash.go create mode 100644 internal/canvasplan/types.go create mode 100644 internal/canvasplan/validate.go diff --git a/internal/canvasplan/decode.go b/internal/canvasplan/decode.go new file mode 100644 index 0000000..7acc15b --- /dev/null +++ b/internal/canvasplan/decode.go @@ -0,0 +1,69 @@ +package canvasplan + +import ( + "encoding/json" + "fmt" + "io" +) + +const maxContractBytes = 16 << 20 + +func DecodePlan(reader io.Reader) (Plan, error) { + var plan Plan + if err := decodeStrictJSON(reader, &plan); err != nil { + return Plan{}, fmt.Errorf("decode CanvasPlan: %w", err) + } + return NormalizePlan(plan) +} + +func DecodeResolvedMedia(reader io.Reader) (ResolvedMediaSet, error) { + var resolved ResolvedMediaSet + if err := decodeStrictJSON(reader, &resolved); err != nil { + return ResolvedMediaSet{}, fmt.Errorf("decode resolved media: %w", err) + } + return NormalizeResolvedMedia(resolved) +} + +func decodeStrictJSON(reader io.Reader, target any) error { + if reader == nil { + return fmt.Errorf("JSON input is required") + } + limited := io.LimitReader(reader, maxContractBytes+1) + payload, err := io.ReadAll(limited) + if err != nil { + return err + } + if len(payload) > maxContractBytes { + return fmt.Errorf("JSON input exceeds %d bytes", maxContractBytes) + } + decoder := json.NewDecoder(bytesReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + return err + } + var trailing any + if err := decoder.Decode(&trailing); err == io.EOF { + return nil + } else if err != nil { + return fmt.Errorf("decode trailing JSON: %w", err) + } + return fmt.Errorf("JSON input must contain exactly one value") +} + +type byteReader struct { + data []byte + off int +} + +func bytesReader(data []byte) *byteReader { + return &byteReader{data: data} +} + +func (r *byteReader) Read(p []byte) (int, error) { + if r.off >= len(r.data) { + return 0, io.EOF + } + n := copy(p, r.data[r.off:]) + r.off += n + return n, nil +} diff --git a/internal/canvasplan/hash.go b/internal/canvasplan/hash.go new file mode 100644 index 0000000..de24d91 --- /dev/null +++ b/internal/canvasplan/hash.go @@ -0,0 +1,40 @@ +package canvasplan + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" +) + +func hashJSON(value any) (string, error) { + payload, err := json.Marshal(value) + if err != nil { + return "", err + } + return hashBytes(payload), nil +} + +func hashRawJSON(raw json.RawMessage) (string, error) { + var value any + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { + return "", fmt.Errorf("decode JSON for hashing: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return "", fmt.Errorf("decode JSON for hashing: multiple values") + } + return "", fmt.Errorf("decode trailing JSON for hashing: %w", err) + } + return hashJSON(value) +} + +func hashBytes(value []byte) string { + digest := sha256.Sum256(value) + return hex.EncodeToString(digest[:]) +} diff --git a/internal/canvasplan/types.go b/internal/canvasplan/types.go new file mode 100644 index 0000000..b24c08d --- /dev/null +++ b/internal/canvasplan/types.go @@ -0,0 +1,199 @@ +package canvasplan + +import ( + "encoding/json" + "time" + + "github.com/Pippit-dev/pippit-cli/internal/canvas" +) + +const ( + PlanSchema = "pippit-canvas-plan/0.1" + ResolvedMediaSchema = "pippit-canvas-resolved-media/0.1" + JournalSchema = "pippit-canvas-execution-journal/0.1" +) + +const ( + StateInitialized = "initialized" + StateCreateRequested = "create-requested" + StateCreatePending = "create-pending" + StateCreateAmbiguous = "create-ambiguous" + StateCreateFailed = "create-failed" + StateRootReady = "root-ready" + StateAllocationRequested = "allocation-requested" + StateAllocated = "allocated" + StateMaterialized = "materialized" + StateApplyPrepared = "apply-prepared" + StateApplyRequested = "apply-requested" + StateApplyAcknowledged = "apply-acknowledged" + StateApplyAmbiguous = "apply-ambiguous" + StateVerificationFailed = "verification-failed" + StateUnsafePartial = "unsafe-partial-apply" + StateUnsafeRootChanged = "unsafe-root-changed" + StateMaterializationDrift = "unsafe-materialization-drift" + StateVerified = "verified" +) + +type Plan struct { + Schema string `json:"schema"` + Title string `json:"title"` + Source Source `json:"source"` + RequiredMedia []MediaRequirement `json:"required_media"` + Nodes []Node `json:"nodes"` + Groups []Group `json:"groups"` + Edges []Edge `json:"edges"` + Degradations []json.RawMessage `json:"degradations,omitempty"` +} + +type Source struct { + Provider string `json:"provider"` + ProjectID string `json:"project_id"` + Fingerprint string `json:"fingerprint"` +} + +type Position struct { + X float64 `json:"x"` + Y float64 `json:"y"` +} + +type Size struct { + Width float64 `json:"width"` + Height float64 `json:"height"` +} + +type MediaMetadata struct { + ByteSize *int64 `json:"byte_size,omitempty"` + DurationMS *int64 `json:"duration_ms,omitempty"` + Extension string `json:"extension,omitempty"` + Height *int64 `json:"height,omitempty"` + MimeType string `json:"mime_type,omitempty"` + Width *int64 `json:"width,omitempty"` +} + +type MediaRequirement struct { + LogicalID string `json:"logical_id"` + SourceNodeID string `json:"source_node_id"` + FileName string `json:"file_name"` + MediaType string `json:"media_type"` + URL string `json:"url,omitempty"` + LocalPath string `json:"local_path,omitempty"` + SHA256 string `json:"sha256,omitempty"` + Metadata MediaMetadata `json:"metadata,omitempty"` +} + +type Node struct { + LogicalID string `json:"logical_id"` + SourceNodeID string `json:"source_node_id"` + Title string `json:"title"` + Position Position `json:"position"` + Size Size `json:"size"` + ParentGroupLogicalID string `json:"parent_group_logical_id,omitempty"` + Order int `json:"order"` + Kind string `json:"kind"` + TargetType string `json:"target_type"` + MediaLogicalID string `json:"media_logical_id,omitempty"` + Variant string `json:"variant,omitempty"` + InputNodeLogicalIDs []string `json:"input_node_logical_ids,omitempty"` +} + +type Group struct { + LogicalID string `json:"logical_id"` + SourceNodeID string `json:"source_node_id"` + Title string `json:"title"` + Position Position `json:"position"` + Size Size `json:"size"` + Order int `json:"order"` + ChildLogicalIDs []string `json:"child_logical_ids"` +} + +type Edge struct { + LogicalID string `json:"logical_id"` + SourceEdgeID string `json:"source_edge_id"` + Type string `json:"type"` + SourceNodeLogicalID string `json:"source_node_logical_id"` + TargetNodeLogicalID string `json:"target_node_logical_id"` + SourceHandle string `json:"source_handle"` + TargetHandle string `json:"target_handle"` +} + +type ResolvedMediaSet struct { + Schema string `json:"schema"` + Media []ResolvedMedia `json:"media"` +} + +type ResolvedMedia struct { + LogicalID string `json:"logical_id"` + MediaType string `json:"media_type"` + AssetID string `json:"asset_id"` + PippitAssetID string `json:"pippit_asset_id"` +} + +type Document struct { + Revision int `json:"revision"` + RootCanvasID string `json:"rootCanvasId"` + Assets map[string]json.RawMessage `json:"assets"` +} + +type Verification struct { + ExpectedAssetCount int `json:"expected_asset_count"` + ReturnedAssetCount int `json:"returned_asset_count"` + MissingAssetIDs []string `json:"missing_asset_ids,omitempty"` + UnverifiableAssetIDs []string `json:"unverifiable_asset_ids,omitempty"` + MismatchedAssetIDs []string `json:"mismatched_asset_ids,omitempty"` + Verified bool `json:"verified"` + RecoveredFromQuery bool `json:"recovered_from_query,omitempty"` + LogID string `json:"log_id,omitempty"` +} + +type ExecuteOptions struct { + JournalPath string + PollInterval time.Duration + WaitTimeout time.Duration +} + +type ExecutionResult struct { + State string `json:"state"` + JournalPath string `json:"journal_path"` + OperationID string `json:"operation_id"` + ProjectID string `json:"project_id,omitempty"` + RootCanvasID string `json:"root_canvas_id,omitempty"` + OverviewPippitAssetID string `json:"overview_pippit_asset_id,omitempty"` + WebURL string `json:"web_url,omitempty"` + DocumentSHA256 string `json:"document_sha256,omitempty"` + AssetCount int `json:"asset_count,omitempty"` + NodeCount int `json:"node_count,omitempty"` + EdgeCount int `json:"edge_count,omitempty"` + TransactionID string `json:"transaction_id,omitempty"` + Verification *Verification `json:"verification,omitempty"` + Warning string `json:"warning,omitempty"` +} + +type Journal struct { + Schema string `json:"schema"` + OperationID string `json:"operation_id"` + RequestID string `json:"request_id"` + PlanSHA256 string `json:"plan_sha256"` + ResolvedMediaSHA256 string `json:"resolved_media_sha256"` + State string `json:"state"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + Create *canvas.CreateResult `json:"create,omitempty"` + NodeAssetIDs map[string]string `json:"node_asset_ids,omitempty"` + AllocationLogID string `json:"allocation_log_id,omitempty"` + DocumentSHA256 string `json:"document_sha256,omitempty"` + AssetSHA256 map[string]string `json:"asset_sha256,omitempty"` + Apply *ApplyJournal `json:"apply,omitempty"` + Verification *Verification `json:"verification,omitempty"` + LastError string `json:"last_error,omitempty"` +} + +type ApplyJournal struct { + TransactionID string `json:"transaction_id"` + BatchID string `json:"batch_id"` + ClientID string `json:"client_id"` + BaseRootVersion int64 `json:"base_root_version"` + RequestSHA256 string `json:"request_sha256"` + Status string `json:"status"` + AssetVersions map[string]int64 `json:"asset_versions,omitempty"` + LogID string `json:"log_id,omitempty"` +} diff --git a/internal/canvasplan/validate.go b/internal/canvasplan/validate.go new file mode 100644 index 0000000..72bb7e3 --- /dev/null +++ b/internal/canvasplan/validate.go @@ -0,0 +1,414 @@ +package canvasplan + +import ( + "encoding/json" + "fmt" + "math" + "net/url" + "path" + "regexp" + "sort" + "strings" + "unicode" +) + +var sha256Pattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +func NormalizePlan(input Plan) (Plan, error) { + plan := input + plan.Schema = strings.TrimSpace(plan.Schema) + plan.Title = strings.TrimSpace(plan.Title) + plan.Source.Provider = strings.TrimSpace(plan.Source.Provider) + plan.Source.ProjectID = strings.TrimSpace(plan.Source.ProjectID) + plan.Source.Fingerprint = strings.TrimSpace(plan.Source.Fingerprint) + if plan.Schema != PlanSchema { + return Plan{}, fmt.Errorf("unsupported CanvasPlan schema %q", plan.Schema) + } + if plan.Title == "" || len([]rune(plan.Title)) > 50 { + return Plan{}, fmt.Errorf("CanvasPlan title must contain 1 to 50 characters") + } + if plan.Source.Provider == "" || plan.Source.ProjectID == "" || plan.Source.Fingerprint == "" { + return Plan{}, fmt.Errorf("CanvasPlan source provider, project_id, and fingerprint are required") + } + if len(plan.Nodes) == 0 { + return Plan{}, fmt.Errorf("CanvasPlan must contain at least one business node") + } + + mediaByID := make(map[string]MediaRequirement, len(plan.RequiredMedia)) + for index := range plan.RequiredMedia { + media := &plan.RequiredMedia[index] + media.LogicalID = strings.TrimSpace(media.LogicalID) + media.SourceNodeID = strings.TrimSpace(media.SourceNodeID) + media.FileName = strings.TrimSpace(media.FileName) + media.MediaType = strings.ToLower(strings.TrimSpace(media.MediaType)) + media.URL = strings.TrimSpace(media.URL) + media.LocalPath = strings.TrimSpace(media.LocalPath) + media.SHA256 = strings.ToLower(strings.TrimSpace(media.SHA256)) + media.Metadata.Extension = strings.ToLower(strings.TrimSpace(media.Metadata.Extension)) + media.Metadata.MimeType = strings.TrimSpace(media.Metadata.MimeType) + if err := validateLogicalID(media.LogicalID, fmt.Sprintf("required_media[%d].logical_id", index)); err != nil { + return Plan{}, err + } + if media.SourceNodeID == "" || media.FileName == "" { + return Plan{}, fmt.Errorf("required_media[%d] source_node_id and file_name are required", index) + } + if media.MediaType != "video" && media.MediaType != "audio" && media.MediaType != "image" { + return Plan{}, fmt.Errorf("required_media[%d].media_type must be video, audio, or image", index) + } + if (media.URL == "") == (media.LocalPath == "") { + return Plan{}, fmt.Errorf("required_media[%d] must define exactly one of url or local_path", index) + } + if media.URL != "" { + parsed, err := url.Parse(media.URL) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" { + return Plan{}, fmt.Errorf("required_media[%d].url must be an absolute HTTPS URL", index) + } + } + if media.LocalPath != "" { + cleaned := path.Clean(media.LocalPath) + if strings.Contains(media.LocalPath, `\`) || cleaned != media.LocalPath || strings.HasPrefix(cleaned, "/") || cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return Plan{}, fmt.Errorf("required_media[%d].local_path must be a bundle-relative POSIX path", index) + } + media.LocalPath = cleaned + if !sha256Pattern.MatchString(media.SHA256) { + return Plan{}, fmt.Errorf("required_media[%d].sha256 must contain 64 lowercase hexadecimal characters for local_path", index) + } + if media.Metadata.ByteSize == nil || *media.Metadata.ByteSize <= 0 { + return Plan{}, fmt.Errorf("required_media[%d].metadata.byte_size must be positive for local_path", index) + } + } else if media.SHA256 != "" && !sha256Pattern.MatchString(media.SHA256) { + return Plan{}, fmt.Errorf("required_media[%d].sha256 must contain 64 lowercase hexadecimal characters", index) + } + if err := validateMediaMetadata(media.Metadata, index); err != nil { + return Plan{}, err + } + if _, duplicate := mediaByID[media.LogicalID]; duplicate { + return Plan{}, fmt.Errorf("duplicate required media logical_id %q", media.LogicalID) + } + mediaByID[media.LogicalID] = *media + } + + nodesByID := make(map[string]Node, len(plan.Nodes)) + allLogicalIDs := make(map[string]struct{}, len(plan.Nodes)+len(plan.Groups)) + for index := range plan.Nodes { + node := &plan.Nodes[index] + normalizeNode(node) + if err := validateLogicalID(node.LogicalID, fmt.Sprintf("nodes[%d].logical_id", index)); err != nil { + return Plan{}, err + } + if node.SourceNodeID == "" || node.Title == "" { + return Plan{}, fmt.Errorf("nodes[%d] source_node_id and title are required", index) + } + if err := validateGeometry(node.Position, node.Size, fmt.Sprintf("nodes[%d]", index)); err != nil { + return Plan{}, err + } + if _, duplicate := allLogicalIDs[node.LogicalID]; duplicate { + return Plan{}, fmt.Errorf("duplicate node logical_id %q", node.LogicalID) + } + allLogicalIDs[node.LogicalID] = struct{}{} + if err := validateNodeContract(*node, mediaByID); err != nil { + return Plan{}, fmt.Errorf("nodes[%d]: %w", index, err) + } + nodesByID[node.LogicalID] = *node + } + + groupsByID := make(map[string]Group, len(plan.Groups)) + for index := range plan.Groups { + group := &plan.Groups[index] + group.LogicalID = strings.TrimSpace(group.LogicalID) + group.SourceNodeID = strings.TrimSpace(group.SourceNodeID) + group.Title = strings.TrimSpace(group.Title) + if err := validateLogicalID(group.LogicalID, fmt.Sprintf("groups[%d].logical_id", index)); err != nil { + return Plan{}, err + } + if group.SourceNodeID == "" || group.Title == "" { + return Plan{}, fmt.Errorf("groups[%d] source_node_id and title are required", index) + } + if err := validateGeometry(group.Position, group.Size, fmt.Sprintf("groups[%d]", index)); err != nil { + return Plan{}, err + } + if _, duplicate := allLogicalIDs[group.LogicalID]; duplicate { + return Plan{}, fmt.Errorf("duplicate node/group logical_id %q", group.LogicalID) + } + allLogicalIDs[group.LogicalID] = struct{}{} + seenChildren := make(map[string]struct{}, len(group.ChildLogicalIDs)) + for childIndex, childID := range group.ChildLogicalIDs { + childID = strings.TrimSpace(childID) + group.ChildLogicalIDs[childIndex] = childID + if err := validateLogicalID(childID, fmt.Sprintf("groups[%d].child_logical_ids[%d]", index, childIndex)); err != nil { + return Plan{}, err + } + if childID == group.LogicalID { + return Plan{}, fmt.Errorf("group %q cannot contain itself", group.LogicalID) + } + if _, duplicate := seenChildren[childID]; duplicate { + return Plan{}, fmt.Errorf("group %q contains duplicate child %q", group.LogicalID, childID) + } + seenChildren[childID] = struct{}{} + } + groupsByID[group.LogicalID] = *group + } + if err := validateGroupGraph(plan.Nodes, plan.Groups, nodesByID, groupsByID); err != nil { + return Plan{}, err + } + + seenEdges := make(map[string]struct{}, len(plan.Edges)) + for index := range plan.Edges { + edge := &plan.Edges[index] + edge.LogicalID = strings.TrimSpace(edge.LogicalID) + edge.SourceEdgeID = strings.TrimSpace(edge.SourceEdgeID) + edge.Type = strings.ToLower(strings.TrimSpace(edge.Type)) + edge.SourceNodeLogicalID = strings.TrimSpace(edge.SourceNodeLogicalID) + edge.TargetNodeLogicalID = strings.TrimSpace(edge.TargetNodeLogicalID) + edge.SourceHandle = strings.TrimSpace(edge.SourceHandle) + edge.TargetHandle = strings.TrimSpace(edge.TargetHandle) + if err := validateLogicalID(edge.LogicalID, fmt.Sprintf("edges[%d].logical_id", index)); err != nil { + return Plan{}, err + } + if _, duplicate := seenEdges[edge.LogicalID]; duplicate { + return Plan{}, fmt.Errorf("duplicate edge logical_id %q", edge.LogicalID) + } + seenEdges[edge.LogicalID] = struct{}{} + if edge.SourceEdgeID == "" || edge.Type != "reference" || edge.SourceHandle == "" || edge.TargetHandle == "" { + return Plan{}, fmt.Errorf("edges[%d] requires source_edge_id, reference type, and handles", index) + } + if _, ok := nodesByID[edge.SourceNodeLogicalID]; !ok { + return Plan{}, fmt.Errorf("edge %q references missing source node %q", edge.LogicalID, edge.SourceNodeLogicalID) + } + if _, ok := nodesByID[edge.TargetNodeLogicalID]; !ok { + return Plan{}, fmt.Errorf("edge %q references missing target node %q", edge.LogicalID, edge.TargetNodeLogicalID) + } + } + + usedMedia := make(map[string]struct{}) + hasPlaceholder := false + for _, node := range plan.Nodes { + if node.MediaLogicalID != "" { + usedMedia[node.MediaLogicalID] = struct{}{} + } + if node.Kind == "image-placeholder" || node.Kind == "video-placeholder" { + hasPlaceholder = true + } + for _, inputID := range node.InputNodeLogicalIDs { + input, ok := nodesByID[inputID] + if !ok || input.Kind != "video" { + return Plan{}, fmt.Errorf("composite node %q input %q must reference a video node", node.LogicalID, inputID) + } + } + } + if len(usedMedia) != len(mediaByID) { + return Plan{}, fmt.Errorf("CanvasPlan required_media must exactly match video/audio node media references") + } + for mediaID := range mediaByID { + if _, used := usedMedia[mediaID]; !used { + return Plan{}, fmt.Errorf("required media %q is not referenced by a node", mediaID) + } + } + for index, degradation := range plan.Degradations { + if len(degradation) == 0 || !json.Valid(degradation) { + return Plan{}, fmt.Errorf("degradations[%d] must be valid JSON", index) + } + var record map[string]json.RawMessage + if err := json.Unmarshal(degradation, &record); err != nil || record == nil { + return Plan{}, fmt.Errorf("degradations[%d] must be a JSON object", index) + } + } + if hasPlaceholder && len(plan.Degradations) == 0 { + return Plan{}, fmt.Errorf("CanvasPlan placeholders require an explicit degradation record") + } + return plan, nil +} + +func NormalizeResolvedMedia(input ResolvedMediaSet) (ResolvedMediaSet, error) { + resolved := input + resolved.Schema = strings.TrimSpace(resolved.Schema) + if resolved.Schema != ResolvedMediaSchema { + return ResolvedMediaSet{}, fmt.Errorf("unsupported resolved media schema %q", resolved.Schema) + } + seenLogical := make(map[string]struct{}, len(resolved.Media)) + for index := range resolved.Media { + item := &resolved.Media[index] + item.LogicalID = strings.TrimSpace(item.LogicalID) + item.MediaType = strings.ToLower(strings.TrimSpace(item.MediaType)) + item.AssetID = strings.TrimSpace(item.AssetID) + item.PippitAssetID = strings.TrimSpace(item.PippitAssetID) + if err := validateLogicalID(item.LogicalID, fmt.Sprintf("media[%d].logical_id", index)); err != nil { + return ResolvedMediaSet{}, err + } + if item.MediaType != "video" && item.MediaType != "audio" && item.MediaType != "image" { + return ResolvedMediaSet{}, fmt.Errorf("media[%d].media_type must be video, audio, or image", index) + } + if item.AssetID == "" || item.PippitAssetID == "" { + return ResolvedMediaSet{}, fmt.Errorf("media[%d] asset_id and pippit_asset_id are required JSON strings", index) + } + if _, duplicate := seenLogical[item.LogicalID]; duplicate { + return ResolvedMediaSet{}, fmt.Errorf("duplicate resolved media logical_id %q", item.LogicalID) + } + seenLogical[item.LogicalID] = struct{}{} + } + sort.Slice(resolved.Media, func(i, j int) bool { return resolved.Media[i].LogicalID < resolved.Media[j].LogicalID }) + return resolved, nil +} + +func ValidateResolution(plan Plan, resolved ResolvedMediaSet) error { + requirements := make(map[string]MediaRequirement, len(plan.RequiredMedia)) + for _, item := range plan.RequiredMedia { + requirements[item.LogicalID] = item + } + if len(requirements) != len(resolved.Media) { + return fmt.Errorf("resolved media count %d does not match required media count %d", len(resolved.Media), len(requirements)) + } + for _, item := range resolved.Media { + requirement, ok := requirements[item.LogicalID] + if !ok { + return fmt.Errorf("resolved media %q is not required by CanvasPlan", item.LogicalID) + } + if requirement.MediaType != item.MediaType { + return fmt.Errorf("resolved media %q type %q does not match required type %q", item.LogicalID, item.MediaType, requirement.MediaType) + } + } + return nil +} + +func normalizeNode(node *Node) { + node.LogicalID = strings.TrimSpace(node.LogicalID) + node.SourceNodeID = strings.TrimSpace(node.SourceNodeID) + node.Title = strings.TrimSpace(node.Title) + node.ParentGroupLogicalID = strings.TrimSpace(node.ParentGroupLogicalID) + node.Kind = strings.ToLower(strings.TrimSpace(node.Kind)) + node.TargetType = strings.TrimSpace(node.TargetType) + node.MediaLogicalID = strings.TrimSpace(node.MediaLogicalID) + node.Variant = strings.TrimSpace(node.Variant) + for index := range node.InputNodeLogicalIDs { + node.InputNodeLogicalIDs[index] = strings.TrimSpace(node.InputNodeLogicalIDs[index]) + } +} + +func validateNodeContract(node Node, mediaByID map[string]MediaRequirement) error { + switch node.Kind { + case "video", "audio", "image": + expectedTarget := "biz/video" + if node.Kind == "audio" { + expectedTarget = "biz/audio" + } else if node.Kind == "image" { + expectedTarget = "biz/image" + } + if node.TargetType != expectedTarget { + return fmt.Errorf("%s node target_type must be %s", node.Kind, expectedTarget) + } + media, ok := mediaByID[node.MediaLogicalID] + if !ok || media.MediaType != node.Kind { + return fmt.Errorf("%s node must reference matching required_media", node.Kind) + } + if node.Variant != "" || len(node.InputNodeLogicalIDs) != 0 { + return fmt.Errorf("uploaded media node cannot define composite fields") + } + case "image-placeholder", "video-placeholder": + expectedTarget := "biz/image" + if node.Kind == "video-placeholder" { + expectedTarget = "biz/video" + } + if node.TargetType != expectedTarget { + return fmt.Errorf("%s node target_type must be %s", node.Kind, expectedTarget) + } + if node.MediaLogicalID != "" || node.Variant != "" || len(node.InputNodeLogicalIDs) != 0 { + return fmt.Errorf("placeholder node cannot define media or composite fields") + } + case "video-composite": + if node.TargetType != "biz/video" || node.Variant != "video-composite" { + return fmt.Errorf("video-composite target_type and variant are invalid") + } + if node.MediaLogicalID != "" { + return fmt.Errorf("video-composite cannot reference required_media") + } + default: + return fmt.Errorf("unsupported node kind %q", node.Kind) + } + return nil +} + +func validateMediaMetadata(metadata MediaMetadata, index int) error { + for name, value := range map[string]*int64{ + "byte_size": metadata.ByteSize, + "duration_ms": metadata.DurationMS, + "height": metadata.Height, + "width": metadata.Width, + } { + if value != nil && *value <= 0 { + return fmt.Errorf("required_media[%d].metadata.%s must be positive when present", index, name) + } + } + return nil +} + +func validateGeometry(position Position, size Size, field string) error { + if math.IsNaN(position.X) || math.IsInf(position.X, 0) || math.IsNaN(position.Y) || math.IsInf(position.Y, 0) { + return fmt.Errorf("%s position must be finite", field) + } + if math.IsNaN(size.Width) || math.IsInf(size.Width, 0) || math.IsNaN(size.Height) || math.IsInf(size.Height, 0) || size.Width <= 0 || size.Height <= 0 { + return fmt.Errorf("%s size must be finite and positive", field) + } + return nil +} + +func validateLogicalID(value, field string) error { + if value == "" || len(value) > 256 || !strings.Contains(value, ":") { + return fmt.Errorf("%s must be a namespaced logical ID", field) + } + for _, char := range value { + if unicode.IsControl(char) || unicode.IsSpace(char) { + return fmt.Errorf("%s must not contain whitespace or control characters", field) + } + } + return nil +} + +func validateGroupGraph(nodes []Node, groups []Group, nodesByID map[string]Node, groupsByID map[string]Group) error { + parentByChild := make(map[string]string) + for _, group := range groups { + for _, childID := range group.ChildLogicalIDs { + if _, node := nodesByID[childID]; !node { + if _, nestedGroup := groupsByID[childID]; !nestedGroup { + return fmt.Errorf("group %q references missing child %q", group.LogicalID, childID) + } + } + if previous, duplicate := parentByChild[childID]; duplicate { + return fmt.Errorf("child %q belongs to both groups %q and %q", childID, previous, group.LogicalID) + } + parentByChild[childID] = group.LogicalID + } + } + for _, node := range nodes { + if inferred := parentByChild[node.LogicalID]; inferred != node.ParentGroupLogicalID { + return fmt.Errorf("node %q parent_group_logical_id %q does not match group children %q", node.LogicalID, node.ParentGroupLogicalID, inferred) + } + } + visiting := make(map[string]bool) + visited := make(map[string]bool) + var visit func(string) error + visit = func(groupID string) error { + if visiting[groupID] { + return fmt.Errorf("group hierarchy contains a cycle at %q", groupID) + } + if visited[groupID] { + return nil + } + visiting[groupID] = true + for _, childID := range groupsByID[groupID].ChildLogicalIDs { + if _, ok := groupsByID[childID]; ok { + if err := visit(childID); err != nil { + return err + } + } + } + visiting[groupID] = false + visited[groupID] = true + return nil + } + for groupID := range groupsByID { + if err := visit(groupID); err != nil { + return err + } + } + return nil +} From 3a9b1c57ac1f3a3e8cfb5f1bb217552b3e4cbc92 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:46:11 +0800 Subject: [PATCH 08/48] feat: materialize CanvasPlan documents Co-authored-by: Codex <codex@openai.com> --- internal/canvasplan/materialize.go | 393 +++++++++++++++++++++++++++++ internal/canvasplan/query.go | 163 ++++++++++++ 2 files changed, 556 insertions(+) create mode 100644 internal/canvasplan/materialize.go create mode 100644 internal/canvasplan/query.go diff --git a/internal/canvasplan/materialize.go b/internal/canvasplan/materialize.go new file mode 100644 index 0000000..a27fd15 --- /dev/null +++ b/internal/canvasplan/materialize.go @@ -0,0 +1,393 @@ +package canvasplan + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strings" +) + +func Materialize( + inputPlan Plan, + inputResolved ResolvedMediaSet, + rootCanvasID string, + nodeAssetIDs map[string]string, +) (*Document, error) { + plan, err := NormalizePlan(inputPlan) + if err != nil { + return nil, err + } + resolved, err := NormalizeResolvedMedia(inputResolved) + if err != nil { + return nil, err + } + if err := ValidateResolution(plan, resolved); err != nil { + return nil, err + } + rootCanvasID = strings.TrimSpace(rootCanvasID) + if rootCanvasID == "" { + return nil, fmt.Errorf("root canvas ID is required") + } + mapping, err := validateNodeAssetIDs(plan, resolved, rootCanvasID, nodeAssetIDs) + if err != nil { + return nil, err + } + resolvedByID := make(map[string]ResolvedMedia, len(resolved.Media)) + for _, item := range resolved.Media { + resolvedByID[item.LogicalID] = item + } + mediaByID := make(map[string]MediaRequirement, len(plan.RequiredMedia)) + for _, item := range plan.RequiredMedia { + mediaByID[item.LogicalID] = item + } + groupParent := groupParents(plan.Groups) + + rootNodes := make(map[string]any, len(plan.Nodes)+len(plan.Groups)) + assets := make(map[string]json.RawMessage, len(plan.Nodes)+1) + for _, group := range plan.Groups { + children := make([]string, 0, len(group.ChildLogicalIDs)) + for _, childID := range group.ChildLogicalIDs { + if assigned, ok := mapping[childID]; ok { + children = append(children, assigned) + } else { + children = append(children, childID) + } + } + rootNodes[group.LogicalID] = map[string]any{ + "id": group.LogicalID, + "type": "group", + "x": group.Position.X, + "y": group.Position.Y, + "w": group.Size.Width, + "h": group.Size.Height, + "parentId": nullableString(groupParent[group.LogicalID]), + "order": group.Order, + "data": map[string]any{ + "title": group.Title, + "collapsed": false, + }, + "children": children, + } + } + + for _, node := range plan.Nodes { + assetID := mapping[node.LogicalID] + projectionData := map[string]any{"name": node.Title} + if node.Kind == "audio" { + projectionData = map[string]any{} + } + rootNodes[assetID] = map[string]any{ + "id": assetID, + "type": node.TargetType, + "x": node.Position.X, + "y": node.Position.Y, + "w": node.Size.Width, + "h": node.Size.Height, + "parentId": nullableString(node.ParentGroupLogicalID), + "order": node.Order, + "data": projectionData, + } + content, err := materializeNodeContent(node, mediaByID, resolvedByID, mapping) + if err != nil { + return nil, err + } + asset := map[string]any{ + "pippitAssetId": assetID, + "type": node.TargetType, + "content": content, + "extra": map[string]any{}, + } + assets[assetID], err = marshalRaw(asset) + if err != nil { + return nil, fmt.Errorf("marshal node asset %q: %w", node.LogicalID, err) + } + } + + rootEdges := make(map[string]any, len(plan.Edges)) + for _, edge := range plan.Edges { + rootEdges[edge.LogicalID] = map[string]any{ + "id": edge.LogicalID, + "type": edge.Type, + "source": mapping[edge.SourceNodeLogicalID], + "target": mapping[edge.TargetNodeLogicalID], + "sourceHandle": edge.SourceHandle, + "targetHandle": edge.TargetHandle, + } + } + bounds, err := globalBounds(plan, groupParent) + if err != nil { + return nil, err + } + rootAsset := map[string]any{ + "pippitAssetId": rootCanvasID, + "type": "canvas", + "content": map[string]any{ + "metadata": map[string]any{ + "title": plan.Title, + "globalBounds": bounds, + }, + "settings": map[string]any{ + "snap": map[string]any{ + "enabled": true, + "threshold": 8, + "targets": []string{"grid", "node"}, + }, + "grid": map[string]any{"size": 28, "visible": true}, + }, + "nodes": rootNodes, + "edges": rootEdges, + }, + "extra": map[string]any{}, + } + assets[rootCanvasID], err = marshalRaw(rootAsset) + if err != nil { + return nil, fmt.Errorf("marshal root canvas asset: %w", err) + } + return &Document{Revision: 0, RootCanvasID: rootCanvasID, Assets: assets}, nil +} + +func DocumentSHA256(document *Document) (string, error) { + if document == nil { + return "", fmt.Errorf("document is required") + } + return hashJSON(document) +} + +func DocumentAssetSHA256(document *Document) (map[string]string, error) { + if document == nil { + return nil, fmt.Errorf("document is required") + } + result := make(map[string]string, len(document.Assets)) + for assetID, asset := range document.Assets { + hash, err := hashRawJSON(asset) + if err != nil { + return nil, fmt.Errorf("hash asset %q: %w", assetID, err) + } + result[assetID] = hash + } + return result, nil +} + +func DocumentAssetIDs(document *Document) []string { + if document == nil { + return nil + } + ids := make([]string, 0, len(document.Assets)) + for assetID := range document.Assets { + if assetID != document.RootCanvasID { + ids = append(ids, assetID) + } + } + sort.Strings(ids) + return append([]string{document.RootCanvasID}, ids...) +} + +func validateNodeAssetIDs(plan Plan, resolved ResolvedMediaSet, rootID string, input map[string]string) (map[string]string, error) { + if len(input) != len(plan.Nodes) { + return nil, fmt.Errorf("node asset ID count %d does not match node count %d", len(input), len(plan.Nodes)) + } + reserved := map[string]struct{}{rootID: {}} + for _, item := range resolved.Media { + reserved[item.PippitAssetID] = struct{}{} + } + for _, group := range plan.Groups { + reserved[group.LogicalID] = struct{}{} + } + for _, edge := range plan.Edges { + reserved[edge.LogicalID] = struct{}{} + } + result := make(map[string]string, len(input)) + for _, node := range plan.Nodes { + assetID := strings.TrimSpace(input[node.LogicalID]) + if assetID == "" { + return nil, fmt.Errorf("node %q has no allocated asset ID", node.LogicalID) + } + if _, collision := reserved[assetID]; collision { + return nil, fmt.Errorf("allocated asset ID %q collides with another document identifier", assetID) + } + reserved[assetID] = struct{}{} + result[node.LogicalID] = assetID + } + return result, nil +} + +func materializeNodeContent( + node Node, + mediaByID map[string]MediaRequirement, + resolvedByID map[string]ResolvedMedia, + mapping map[string]string, +) (map[string]any, error) { + switch node.Kind { + case "video", "audio", "image": + requirement := mediaByID[node.MediaLogicalID] + resolved := resolvedByID[node.MediaLogicalID] + if node.Kind == "audio" { + content := map[string]any{ + "assetId": resolved.AssetID, + "generation": map[string]any{"source": "uploaded"}, + "name": node.Title, + "pippitAssetId": resolved.PippitAssetID, + "source": "uploaded", + } + putInt64(content, "durationMs", requirement.Metadata.DurationMS) + return content, nil + } + if node.Kind == "image" { + metadata := map[string]any{ + "format": mediaFormat(requirement, "image"), + "name": node.Title, + } + putInt64(metadata, "height", requirement.Metadata.Height) + putInt64(metadata, "size", requirement.Metadata.ByteSize) + putInt64(metadata, "width", requirement.Metadata.Width) + content := map[string]any{ + "assetId": resolved.AssetID, + "generation": map[string]any{"source": "uploaded"}, + "metadata": metadata, + "name": node.Title, + "pippitAssetId": resolved.PippitAssetID, + "source": "uploaded", + "sourceType": 4, + } + putInt64(content, "naturalHeight", requirement.Metadata.Height) + putInt64(content, "naturalWidth", requirement.Metadata.Width) + return content, nil + } + metadata := map[string]any{ + "format": mediaFormat(requirement, "video"), + "name": node.Title, + } + putInt64(metadata, "durationMS", requirement.Metadata.DurationMS) + putInt64(metadata, "durationMs", requirement.Metadata.DurationMS) + putInt64(metadata, "height", requirement.Metadata.Height) + putInt64(metadata, "size", requirement.Metadata.ByteSize) + putInt64(metadata, "width", requirement.Metadata.Width) + content := map[string]any{ + "assetId": resolved.AssetID, + "caption": node.Title, + "metadata": metadata, + "name": node.Title, + "pippitAssetId": resolved.PippitAssetID, + "playback": map[string]any{"muted": true}, + "source": "uploaded", + "title": node.Title, + } + if requirement.Metadata.DurationMS != nil { + content["duration"] = float64(*requirement.Metadata.DurationMS) / 1000 + } + putInt64(content, "durationMs", requirement.Metadata.DurationMS) + putInt64(content, "frameHeight", requirement.Metadata.Height) + putInt64(content, "frameWidth", requirement.Metadata.Width) + putInt64(content, "height", requirement.Metadata.Height) + putInt64(content, "width", requirement.Metadata.Width) + return content, nil + case "image-placeholder": + return map[string]any{"caption": node.Title, "name": node.Title}, nil + case "video-placeholder": + return map[string]any{"caption": node.Title, "name": node.Title, "title": node.Title}, nil + case "video-composite": + references := make([]map[string]any, 0, len(node.InputNodeLogicalIDs)) + for _, inputID := range node.InputNodeLogicalIDs { + assignedID := mapping[inputID] + if assignedID == "" { + return nil, fmt.Errorf("composite node %q input %q has no allocated ID", node.LogicalID, inputID) + } + references = append(references, map[string]any{ + "id": assignedID, + "type": "video", + "nodeAssetId": assignedID, + "sourceNodeId": assignedID, + "sourceNodeType": "biz/video", + }) + } + return map[string]any{ + "caption": "视频", + "generation": map[string]any{ + "source": "tool", + "references": references, + }, + "name": node.Title, + "variant": "video-composite", + }, nil + default: + return nil, fmt.Errorf("unsupported node kind %q", node.Kind) + } +} + +func mediaFormat(requirement MediaRequirement, fallback string) string { + if value := strings.TrimSpace(requirement.Metadata.Extension); value != "" { + return strings.ToLower(strings.TrimPrefix(value, ".")) + } + if extensionIndex := strings.LastIndex(requirement.FileName, "."); extensionIndex >= 0 && extensionIndex < len(requirement.FileName)-1 { + return strings.ToLower(requirement.FileName[extensionIndex+1:]) + } + if slashIndex := strings.Index(requirement.Metadata.MimeType, "/"); slashIndex >= 0 && slashIndex < len(requirement.Metadata.MimeType)-1 { + return strings.ToLower(requirement.Metadata.MimeType[slashIndex+1:]) + } + return fallback +} + +func globalBounds(plan Plan, groupParent map[string]string) (map[string]float64, error) { + type item struct { + position Position + size Size + } + items := make([]item, 0, len(plan.Nodes)+len(plan.Groups)) + for _, node := range plan.Nodes { + if node.ParentGroupLogicalID == "" { + items = append(items, item{position: node.Position, size: node.Size}) + } + } + for _, group := range plan.Groups { + if groupParent[group.LogicalID] == "" { + items = append(items, item{position: group.Position, size: group.Size}) + } + } + if len(items) == 0 { + return nil, fmt.Errorf("CanvasPlan has no top-level layout items") + } + minX, minY := math.Inf(1), math.Inf(1) + maxX, maxY := math.Inf(-1), math.Inf(-1) + for _, item := range items { + minX = math.Min(minX, item.position.X) + minY = math.Min(minY, item.position.Y) + maxX = math.Max(maxX, item.position.X+item.size.Width) + maxY = math.Max(maxY, item.position.Y+item.size.Height) + } + return map[string]float64{"minX": minX, "minY": minY, "maxX": maxX, "maxY": maxY}, nil +} + +func groupParents(groups []Group) map[string]string { + parents := make(map[string]string) + groupIDs := make(map[string]struct{}, len(groups)) + for _, group := range groups { + groupIDs[group.LogicalID] = struct{}{} + } + for _, group := range groups { + for _, childID := range group.ChildLogicalIDs { + if _, isGroup := groupIDs[childID]; isGroup { + parents[childID] = group.LogicalID + } + } + } + return parents +} + +func nullableString(value string) any { + if value == "" { + return nil + } + return value +} + +func putInt64(target map[string]any, key string, value *int64) { + if value != nil { + target[key] = *value + } +} + +func marshalRaw(value any) (json.RawMessage, error) { + payload, err := json.Marshal(value) + return json.RawMessage(payload), err +} diff --git a/internal/canvasplan/query.go b/internal/canvasplan/query.go new file mode 100644 index 0000000..47cd57a --- /dev/null +++ b/internal/canvasplan/query.go @@ -0,0 +1,163 @@ +package canvasplan + +import ( + "encoding/json" + "fmt" + "sort" + "strings" +) + +func VerifyDocument(document *Document, assets []json.RawMessage) Verification { + verification := Verification{ExpectedAssetCount: len(document.Assets), ReturnedAssetCount: len(assets)} + queried := make(map[string]json.RawMessage, len(assets)) + for _, asset := range assets { + assetID, err := queriedAssetID(asset) + if err == nil { + queried[assetID] = asset + } + } + for _, assetID := range DocumentAssetIDs(document) { + queriedAsset, exists := queried[assetID] + if !exists { + verification.MissingAssetIDs = append(verification.MissingAssetIDs, assetID) + continue + } + stored, err := queriedAssetContent(queriedAsset) + if err != nil { + verification.UnverifiableAssetIDs = append(verification.UnverifiableAssetIDs, assetID) + continue + } + expectedHash, expectedErr := hashRawJSON(document.Assets[assetID]) + storedHash, storedErr := hashRawJSON(stored) + if expectedErr != nil || storedErr != nil { + verification.UnverifiableAssetIDs = append(verification.UnverifiableAssetIDs, assetID) + continue + } + if expectedHash != storedHash { + verification.MismatchedAssetIDs = append(verification.MismatchedAssetIDs, assetID) + } + } + verification.Verified = len(verification.MissingAssetIDs) == 0 && + len(verification.UnverifiableAssetIDs) == 0 && + len(verification.MismatchedAssetIDs) == 0 + return verification +} + +func queriedAssetID(raw json.RawMessage) (string, error) { + asset, err := rawObject(raw) + if err != nil { + return "", err + } + value, ok := firstRawField(asset, "PippitAssetID", "pippit_asset_id", "pippitAssetId") + if !ok { + return "", fmt.Errorf("queried asset has no pippit_asset_id") + } + var assetID string + if err := json.Unmarshal(value, &assetID); err != nil { + return "", fmt.Errorf("queried pippit_asset_id must be a JSON string") + } + assetID = strings.TrimSpace(assetID) + if assetID == "" { + return "", fmt.Errorf("queried pippit_asset_id is empty") + } + return assetID, nil +} + +func queriedAssetVersion(raw json.RawMessage) (int64, error) { + asset, err := rawObject(raw) + if err != nil { + return 0, err + } + value, ok := firstRawField(asset, "version", "Version", "asset_version", "assetVersion") + if !ok { + return 0, fmt.Errorf("queried asset has no version") + } + var version int64 + if err := json.Unmarshal(value, &version); err != nil || version < 0 { + return 0, fmt.Errorf("queried asset version must be a non-negative integer") + } + return version, nil +} + +func queriedAssetContent(raw json.RawMessage) (json.RawMessage, error) { + asset, err := rawObject(raw) + if err != nil { + return nil, err + } + textRaw, ok := firstRawField(asset, "TextInfo", "textInfo", "text") + if !ok { + return nil, fmt.Errorf("queried asset has no text content") + } + text, err := rawObject(textRaw) + if err != nil { + return nil, fmt.Errorf("decode queried text info: %w", err) + } + contentRaw, ok := firstRawField(text, "Content", "content") + if !ok { + return nil, fmt.Errorf("queried asset text has no content") + } + var encoded string + if json.Unmarshal(contentRaw, &encoded) == nil { + encoded = strings.TrimSpace(encoded) + if encoded == "" || !json.Valid([]byte(encoded)) { + return nil, fmt.Errorf("queried asset text content is not JSON") + } + return json.RawMessage(encoded), nil + } + if !json.Valid(contentRaw) || string(contentRaw) == "null" { + return nil, fmt.Errorf("queried asset text content is not JSON") + } + return contentRaw, nil +} + +func indexQueriedAssets(assets []json.RawMessage) (map[string]json.RawMessage, error) { + result := make(map[string]json.RawMessage, len(assets)) + for _, asset := range assets { + assetID, err := queriedAssetID(asset) + if err != nil { + return nil, err + } + if _, duplicate := result[assetID]; duplicate { + return nil, fmt.Errorf("query returned duplicate asset %q", assetID) + } + result[assetID] = asset + } + return result, nil +} + +func existingCompanionIDs(document *Document, queried map[string]json.RawMessage) []string { + result := make([]string, 0) + for assetID := range document.Assets { + if assetID == document.RootCanvasID { + continue + } + if _, exists := queried[assetID]; exists { + result = append(result, assetID) + } + } + sort.Strings(result) + return result +} + +func rawObject(raw json.RawMessage) (map[string]json.RawMessage, error) { + var value map[string]json.RawMessage + if len(raw) == 0 || string(raw) == "null" { + return nil, fmt.Errorf("JSON object is missing") + } + if err := json.Unmarshal(raw, &value); err != nil { + return nil, err + } + if value == nil { + return nil, fmt.Errorf("JSON object is missing") + } + return value, nil +} + +func firstRawField(values map[string]json.RawMessage, keys ...string) (json.RawMessage, bool) { + for _, key := range keys { + if value, ok := values[key]; ok { + return value, true + } + } + return nil, false +} From 6e7289ece235ba1c30a711c66203acbe74176a9f Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:46:27 +0800 Subject: [PATCH 09/48] feat: add secure CanvasPlan journals Co-authored-by: Codex <codex@openai.com> --- internal/canvasplan/journal.go | 272 ++++++++++++++++++++ internal/canvasplan/journal_lock_unix.go | 38 +++ internal/canvasplan/journal_lock_windows.go | 89 +++++++ internal/canvasplan/journal_path.go | 139 ++++++++++ 4 files changed, 538 insertions(+) create mode 100644 internal/canvasplan/journal.go create mode 100644 internal/canvasplan/journal_lock_unix.go create mode 100644 internal/canvasplan/journal_lock_windows.go create mode 100644 internal/canvasplan/journal_path.go diff --git a/internal/canvasplan/journal.go b/internal/canvasplan/journal.go new file mode 100644 index 0000000..1ded5a8 --- /dev/null +++ b/internal/canvasplan/journal.go @@ -0,0 +1,272 @@ +package canvasplan + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +const maxJournalBytes = 16 << 20 + +type journalLock struct { + path string + file *os.File +} + +func acquireJournalLock(path string) (*journalLock, error) { + path, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("resolve CanvasPlan journal lock path: %w", err) + } + directory, err := ensureSecureJournalDirectory(filepath.Dir(path)) + if err != nil { + return nil, err + } + lockPath := path + ".lock" + file, err := openRegularFileNoFollow(lockPath, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open CanvasPlan journal lock: %w", err) + } + closeOnError := func(cause error) (*journalLock, error) { + _ = file.Close() + return nil, cause + } + if err := directory.validateStable(); err != nil { + return closeOnError(err) + } + if err := lockJournalFile(file); err != nil { + return closeOnError(fmt.Errorf("CanvasPlan journal is locked: %s: %w", lockPath, err)) + } + if err := file.Chmod(0o600); err != nil { + _ = unlockJournalFile(file) + return closeOnError(fmt.Errorf("secure CanvasPlan journal lock: %w", err)) + } + if err := file.Truncate(0); err != nil { + _ = unlockJournalFile(file) + return closeOnError(fmt.Errorf("truncate CanvasPlan journal lock: %w", err)) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + _ = unlockJournalFile(file) + return closeOnError(fmt.Errorf("seek CanvasPlan journal lock: %w", err)) + } + if _, err := fmt.Fprintf(file, "pid=%d acquired_at=%s\n", os.Getpid(), time.Now().UTC().Format(time.RFC3339Nano)); err != nil { + _ = unlockJournalFile(file) + return closeOnError(fmt.Errorf("write CanvasPlan journal lock: %w", err)) + } + if err := directory.validateStable(); err != nil { + _ = unlockJournalFile(file) + return closeOnError(err) + } + return &journalLock{path: lockPath, file: file}, nil +} + +func (lock *journalLock) release() error { + if lock == nil { + return nil + } + if lock.file != nil { + unlockErr := unlockJournalFile(lock.file) + closeErr := lock.file.Close() + if unlockErr != nil { + return unlockErr + } + return closeErr + } + return nil +} + +func loadOrCreateJournal(path, planHash, resolvedHash string, allowCreate bool) (*Journal, bool, error) { + path, err := filepath.Abs(path) + if err != nil { + return nil, false, fmt.Errorf("resolve CanvasPlan journal path: %w", err) + } + directory, err := ensureSecureJournalDirectory(filepath.Dir(path)) + if err != nil { + return nil, false, err + } + file, err := openRegularFileNoFollow(path, os.O_RDWR, 0) + if err == nil { + defer file.Close() + if err := file.Chmod(0o600); err != nil { + return nil, false, fmt.Errorf("secure CanvasPlan journal permissions: %w", err) + } + journal, err := decodeJournal(io.LimitReader(file, maxJournalBytes+1)) + if err != nil { + return nil, false, err + } + if err := validateJournal(journal, planHash, resolvedHash); err != nil { + return nil, false, err + } + if err := directory.validateStable(); err != nil { + return nil, false, err + } + return journal, false, nil + } + if !os.IsNotExist(err) { + return nil, false, fmt.Errorf("open CanvasPlan journal: %w", err) + } + if !allowCreate { + return nil, false, fmt.Errorf("CanvasPlan journal does not exist: %s", path) + } + operationID, err := randomOperationID() + if err != nil { + return nil, false, err + } + now := time.Now().UTC().Format(time.RFC3339Nano) + journal := &Journal{ + Schema: JournalSchema, + OperationID: operationID, + RequestID: "pippit_canvas_plan_" + operationID, + PlanSHA256: planHash, + ResolvedMediaSHA256: resolvedHash, + State: StateInitialized, + CreatedAt: now, + UpdatedAt: now, + } + if err := saveJournal(path, journal); err != nil { + return nil, false, err + } + return journal, true, nil +} + +func decodeJournal(reader io.Reader) (*Journal, error) { + payload, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read CanvasPlan journal: %w", err) + } + if len(payload) > maxJournalBytes { + return nil, fmt.Errorf("CanvasPlan journal exceeds %d bytes", maxJournalBytes) + } + var journal Journal + if err := json.Unmarshal(payload, &journal); err != nil { + return nil, fmt.Errorf("decode CanvasPlan journal: %w", err) + } + return &journal, nil +} + +func validateJournal(journal *Journal, planHash, resolvedHash string) error { + if journal == nil || journal.Schema != JournalSchema { + return fmt.Errorf("unsupported CanvasPlan journal schema %q", valueOrEmpty(journal, func(j *Journal) string { return j.Schema })) + } + if strings.TrimSpace(journal.OperationID) == "" || strings.TrimSpace(journal.RequestID) == "" || strings.TrimSpace(journal.State) == "" { + return fmt.Errorf("CanvasPlan journal identity or state is incomplete") + } + if journal.PlanSHA256 != planHash || journal.ResolvedMediaSHA256 != resolvedHash { + return fmt.Errorf("CanvasPlan or resolved media changed after journal creation") + } + return nil +} + +func saveJournal(path string, journal *Journal) error { + if journal == nil { + return fmt.Errorf("CanvasPlan journal is required") + } + journal.UpdatedAt = time.Now().UTC().Format(time.RFC3339Nano) + payload, err := json.MarshalIndent(journal, "", " ") + if err != nil { + return fmt.Errorf("encode CanvasPlan journal: %w", err) + } + payload = append(payload, '\n') + path, err = filepath.Abs(path) + if err != nil { + return fmt.Errorf("resolve CanvasPlan journal path: %w", err) + } + directory, err := ensureSecureJournalDirectory(filepath.Dir(path)) + if err != nil { + return err + } + destinationBefore, err := lstatRegularOrMissing(path) + if err != nil { + return fmt.Errorf("inspect CanvasPlan journal destination: %w", err) + } + temporary, err := os.CreateTemp(directory.path, ".canvas-plan-journal-*") + if err != nil { + return fmt.Errorf("create temporary CanvasPlan journal: %w", err) + } + temporaryPath := temporary.Name() + removeTemporary := true + defer func() { + _ = temporary.Close() + if removeTemporary { + _ = os.Remove(temporaryPath) + } + }() + temporaryInfo, err := os.Lstat(temporaryPath) + if err != nil { + return fmt.Errorf("inspect temporary CanvasPlan journal: %w", err) + } + if err := validateOpenedRegularFile(temporaryPath, temporary, temporaryInfo); err != nil { + return fmt.Errorf("validate temporary CanvasPlan journal: %w", err) + } + if err := temporary.Chmod(0o600); err != nil { + return fmt.Errorf("secure temporary CanvasPlan journal: %w", err) + } + if _, err := temporary.Write(payload); err != nil { + return fmt.Errorf("write temporary CanvasPlan journal: %w", err) + } + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync temporary CanvasPlan journal: %w", err) + } + if err := directory.validateStable(); err != nil { + return err + } + if err := validateDestinationUnchanged(path, destinationBefore); err != nil { + return err + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close temporary CanvasPlan journal: %w", err) + } + if err := directory.validateStable(); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return fmt.Errorf("replace CanvasPlan journal: %w", err) + } + replaced, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("inspect replaced CanvasPlan journal: %w", err) + } + if fileInfoIsLinkLike(replaced) || !replaced.Mode().IsRegular() || !os.SameFile(temporaryInfo, replaced) { + return fmt.Errorf("replaced CanvasPlan journal is not the secured temporary file") + } + if err := directory.validateStable(); err != nil { + return err + } + removeTemporary = false + return nil +} + +func validateDestinationUnchanged(path string, before os.FileInfo) error { + current, err := lstatRegularOrMissing(path) + if err != nil { + return fmt.Errorf("reinspect CanvasPlan journal destination: %w", err) + } + if before == nil && current == nil { + return nil + } + if before == nil || current == nil || !os.SameFile(before, current) { + return fmt.Errorf("CanvasPlan journal destination changed during save") + } + return nil +} + +func randomOperationID() (string, error) { + value := make([]byte, 16) + if _, err := rand.Read(value); err != nil { + return "", fmt.Errorf("generate CanvasPlan operation ID: %w", err) + } + return hex.EncodeToString(value), nil +} + +func valueOrEmpty[T any](value *T, getter func(*T) string) string { + if value == nil { + return "" + } + return getter(value) +} diff --git a/internal/canvasplan/journal_lock_unix.go b/internal/canvasplan/journal_lock_unix.go new file mode 100644 index 0000000..0c3f3fe --- /dev/null +++ b/internal/canvasplan/journal_lock_unix.go @@ -0,0 +1,38 @@ +//go:build !windows + +package canvasplan + +import ( + "os" + "path/filepath" + "syscall" +) + +func openFileNoFollow(path string, flags int, mode os.FileMode) (*os.File, error) { + fd, err := syscall.Open(path, flags|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, uint32(mode.Perm())) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(fd), path), nil +} + +func fileInfoIsLinkLike(info os.FileInfo) bool { + return info != nil && info.Mode()&os.ModeSymlink != 0 +} + +func trustedSystemAncestorSymlink(path string, info os.FileInfo) bool { + parent := filepath.Dir(path) + if filepath.Dir(parent) != parent { + return false + } + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && stat.Uid == 0 +} + +func lockJournalFile(file *os.File) error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) +} + +func unlockJournalFile(file *os.File) error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_UN) +} diff --git a/internal/canvasplan/journal_lock_windows.go b/internal/canvasplan/journal_lock_windows.go new file mode 100644 index 0000000..d7f25b2 --- /dev/null +++ b/internal/canvasplan/journal_lock_windows.go @@ -0,0 +1,89 @@ +//go:build windows + +package canvasplan + +import ( + "fmt" + "os" + "syscall" + + "golang.org/x/sys/windows" +) + +func fileInfoIsLinkLike(info os.FileInfo) bool { + if info == nil { + return false + } + if info.Mode()&os.ModeSymlink != 0 { + return true + } + data, ok := info.Sys().(*syscall.Win32FileAttributeData) + return ok && data.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 +} + +func openFileNoFollow(path string, flags int, _ os.FileMode) (*os.File, error) { + pathPtr, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + access := uint32(windows.GENERIC_READ) + if flags&os.O_WRONLY != 0 { + access = windows.GENERIC_WRITE + } else if flags&os.O_RDWR != 0 { + access = windows.GENERIC_READ | windows.GENERIC_WRITE + } + creation := uint32(windows.OPEN_EXISTING) + switch { + case flags&os.O_CREATE != 0 && flags&os.O_EXCL != 0: + creation = windows.CREATE_NEW + case flags&os.O_CREATE != 0 && flags&os.O_TRUNC != 0: + creation = windows.CREATE_ALWAYS + case flags&os.O_CREATE != 0: + creation = windows.OPEN_ALWAYS + case flags&os.O_TRUNC != 0: + creation = windows.TRUNCATE_EXISTING + } + handle, err := windows.CreateFile( + pathPtr, + access, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + creation, + windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return nil, err + } + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + _ = windows.CloseHandle(handle) + return nil, err + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + _ = windows.CloseHandle(handle) + return nil, fmt.Errorf("refusing Windows reparse point: %s", path) + } + return os.NewFile(uintptr(handle), path), nil +} + +func trustedSystemAncestorSymlink(string, os.FileInfo) bool { + return false +} + +func lockJournalFile(file *os.File) error { + var overlapped windows.Overlapped + return windows.LockFileEx( + windows.Handle(file.Fd()), + windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, + 0, + 1, + 0, + &overlapped, + ) +} + +func unlockJournalFile(file *os.File) error { + var overlapped windows.Overlapped + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, &overlapped) +} diff --git a/internal/canvasplan/journal_path.go b/internal/canvasplan/journal_path.go new file mode 100644 index 0000000..0f843f5 --- /dev/null +++ b/internal/canvasplan/journal_path.go @@ -0,0 +1,139 @@ +package canvasplan + +import ( + "fmt" + "os" + "path/filepath" +) + +type secureJournalDirectory struct { + path string + info os.FileInfo +} + +func ensureSecureJournalDirectory(path string) (*secureJournalDirectory, error) { + absolute, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("resolve CanvasPlan journal directory: %w", err) + } + absolute = filepath.Clean(absolute) + components := parentComponents(absolute) + for index, component := range components { + info, lstatErr := os.Lstat(component) + if os.IsNotExist(lstatErr) { + if err := os.Mkdir(component, 0o700); err != nil && !os.IsExist(err) { + return nil, fmt.Errorf("create CanvasPlan journal directory %q: %w", component, err) + } + info, lstatErr = os.Lstat(component) + } + if lstatErr != nil { + return nil, fmt.Errorf("inspect CanvasPlan journal directory %q: %w", component, lstatErr) + } + if fileInfoIsLinkLike(info) { + isFinal := index == len(components)-1 + if isFinal || !trustedSystemAncestorSymlink(component, info) { + return nil, fmt.Errorf("CanvasPlan journal directory must not contain symbolic links: %s", component) + } + continue + } + if !info.IsDir() { + return nil, fmt.Errorf("CanvasPlan journal parent is not a directory: %s", component) + } + } + info, err := os.Lstat(absolute) + if err != nil { + return nil, fmt.Errorf("inspect CanvasPlan journal directory: %w", err) + } + if fileInfoIsLinkLike(info) || !info.IsDir() { + return nil, fmt.Errorf("CanvasPlan journal parent must be a real directory: %s", absolute) + } + return &secureJournalDirectory{path: absolute, info: info}, nil +} + +func parentComponents(path string) []string { + components := make([]string, 0, 8) + for current := filepath.Clean(path); ; current = filepath.Dir(current) { + components = append(components, current) + parent := filepath.Dir(current) + if parent == current { + break + } + } + for left, right := 0, len(components)-1; left < right; left, right = left+1, right-1 { + components[left], components[right] = components[right], components[left] + } + return components +} + +func (directory *secureJournalDirectory) validateStable() error { + if directory == nil || directory.info == nil { + return fmt.Errorf("CanvasPlan journal directory identity is missing") + } + current, err := os.Lstat(directory.path) + if err != nil { + return fmt.Errorf("reinspect CanvasPlan journal directory: %w", err) + } + if fileInfoIsLinkLike(current) || !current.IsDir() || !os.SameFile(directory.info, current) { + return fmt.Errorf("CanvasPlan journal directory changed during operation") + } + return nil +} + +func openRegularFileNoFollow(path string, flags int, mode os.FileMode) (*os.File, error) { + before, err := os.Lstat(path) + if err != nil && !(os.IsNotExist(err) && flags&os.O_CREATE != 0) { + return nil, err + } + if err == nil { + if fileInfoIsLinkLike(before) { + return nil, fmt.Errorf("refusing symbolic link: %s", path) + } + if !before.Mode().IsRegular() { + return nil, fmt.Errorf("refusing non-regular file: %s", path) + } + } + file, err := openFileNoFollow(path, flags, mode) + if err != nil { + return nil, err + } + if err := validateOpenedRegularFile(path, file, before); err != nil { + _ = file.Close() + return nil, err + } + return file, nil +} + +func validateOpenedRegularFile(path string, file *os.File, before os.FileInfo) error { + opened, err := file.Stat() + if err != nil { + return fmt.Errorf("inspect opened file: %w", err) + } + current, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("inspect opened file path: %w", err) + } + if !opened.Mode().IsRegular() || fileInfoIsLinkLike(current) || !current.Mode().IsRegular() || !os.SameFile(opened, current) { + return fmt.Errorf("opened CanvasPlan path is not the expected regular file: %s", path) + } + if before != nil && !os.SameFile(before, current) { + return fmt.Errorf("CanvasPlan file changed while it was being opened: %s", path) + } + return nil +} + +func lstatRegularOrMissing(path string) (os.FileInfo, error) { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + if fileInfoIsLinkLike(info) { + return nil, fmt.Errorf("refusing symbolic link: %s", path) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("refusing non-regular file: %s", path) + } + return info, nil +} From ff8caeb883f9308ac97ce996bddbd2231a248666 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:46:37 +0800 Subject: [PATCH 10/48] feat: execute CanvasPlan transactions safely Co-authored-by: Codex <codex@openai.com> --- internal/canvas/apply.go | 4 +- internal/canvas/canvas_test.go | 2 +- internal/canvas/import_facade.go | 84 +++++ internal/canvasplan/api.go | 89 +++++ internal/canvasplan/executor.go | 570 +++++++++++++++++++++++++++++++ 5 files changed, 746 insertions(+), 3 deletions(-) create mode 100644 internal/canvas/import_facade.go create mode 100644 internal/canvasplan/api.go create mode 100644 internal/canvasplan/executor.go diff --git a/internal/canvas/apply.go b/internal/canvas/apply.go index de53a25..23720bc 100644 --- a/internal/canvas/apply.go +++ b/internal/canvas/apply.go @@ -86,14 +86,14 @@ func Apply(ctx context.Context, opts ApplyOptions, runner *common.Runner) (*Appl var data applyData if err := json.Unmarshal(envelope.Data, &data); err != nil { return nil, common.NewLogIDError( - fmt.Sprintf("canvas apply returned invalid data: %v; query affected assets before retrying because outcome cannot be confirmed", err), + fmt.Sprintf("canvas apply returned invalid data: %v; outcome cannot be verified, query affected asset IDs before retrying and do not replay blindly", err), envelope.LogID, ) } ordered, err := validateApplyResults(request.Transactions, data.Results) if err != nil { return nil, common.NewLogIDError( - fmt.Sprintf("%s; query affected assets before retrying because outcome cannot be confirmed", err), + fmt.Sprintf("%s; outcome cannot be verified, query affected asset IDs before retrying and do not replay blindly", err), envelope.LogID, ) } diff --git a/internal/canvas/canvas_test.go b/internal/canvas/canvas_test.go index 60c3270..89b5008 100644 --- a/internal/canvas/canvas_test.go +++ b/internal/canvas/canvas_test.go @@ -331,7 +331,7 @@ func TestApplyRequiresAcknowledgementAndEveryAssetVersion(t *testing.T) { return decodeInto(out, `{"ret":"0","data":{"results":[{"transaction_id":"tx-1","status":"ack","asset_versions":{}}]}}`) }} _, err = Apply(context.Background(), ApplyOptions{Request: request}, runnerWithClient(missingVersion)) - if err == nil || !strings.Contains(err.Error(), "omitted version") { + if err == nil || !strings.Contains(err.Error(), "omitted version") || !strings.Contains(err.Error(), "do not replay blindly") { t.Fatalf("Apply() error = %v, want missing version rejection", err) } } diff --git a/internal/canvas/import_facade.go b/internal/canvas/import_facade.go new file mode 100644 index 0000000..254ad30 --- /dev/null +++ b/internal/canvas/import_facade.go @@ -0,0 +1,84 @@ +package canvas + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +// ResumeCreateOptions controls polling an already accepted create operation. +// It never creates a second project. +type ResumeCreateOptions struct { + PollInterval time.Duration + WaitTimeout time.Duration +} + +// ResumeCreate waits for an already accepted create operation and preserves +// its IDs on transient polling failures. Callers must not use it without a +// previously returned CreateResult. +func ResumeCreate( + ctx context.Context, + accepted *CreateResult, + opts ResumeCreateOptions, + runner *common.Runner, +) (*CreateResult, error) { + if accepted == nil { + return nil, fmt.Errorf("accepted canvas create result is required") + } + result := *accepted + result.Warning = "" + if err := validateCreateResult(&result); err != nil { + return nil, err + } + if opts.PollInterval < 0 || opts.WaitTimeout < 0 { + return nil, fmt.Errorf("canvas create polling durations must not be negative") + } + if result.OverviewPippitAssetID != "" { + return finalizeReadyCanvas(ctx, &result, runner) + } + artifact, attempts, waitErr := waitForCreationArtifact( + ctx, + runner, + result.ThreadID, + result.RunID, + opts.PollInterval, + opts.WaitTimeout, + ) + result.PollAttempts += attempts + if waitErr != nil { + result.Warning = waitErr.Error() + var terminal *CreationTerminalError + if errors.As(waitErr, &terminal) { + result.State = "failed" + return &result, acceptedCreationError(&result, waitErr) + } + result.State = StateCreating + return &result, nil + } + if artifact.CanvasAssetID != "" && artifact.CanvasAssetID != result.CanvasAssetID { + result.State = "failed" + result.Warning = fmt.Sprintf( + "canvas create artifact canvas_asset_id mismatch: got %q, want %q", + artifact.CanvasAssetID, + result.CanvasAssetID, + ) + return &result, acceptedCreationError(&result, errors.New(result.Warning)) + } + result.OverviewPippitAssetID = strings.TrimSpace(artifact.OverviewPippitAssetID) + return finalizeReadyCanvas(ctx, &result, runner) +} + +// GetExisting returns the requested assets that currently exist without +// treating absent IDs as a protocol error. The response envelope and every +// returned ID remain strictly validated. +func GetExisting(ctx context.Context, assetIDs []string, runner *common.Runner) (*GetResult, error) { + normalized, err := normalizeAssetIDs(assetIDs) + if err != nil { + return nil, err + } + return queryAssets(ctx, normalized, false, runner) +} diff --git a/internal/canvasplan/api.go b/internal/canvasplan/api.go new file mode 100644 index 0000000..70d98f4 --- /dev/null +++ b/internal/canvasplan/api.go @@ -0,0 +1,89 @@ +package canvasplan + +import ( + "context" + "fmt" + + "github.com/Pippit-dev/pippit-cli/internal/canvas" + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +type canvasAPI interface { + Create(context.Context, canvas.CreateOptions) (*canvas.CreateResult, error) + ResumeCreate(context.Context, *canvas.CreateResult, canvas.ResumeCreateOptions) (*canvas.CreateResult, error) + Allocate(context.Context, int) (*canvas.AllocateResult, error) + Get(context.Context, []string) (*canvas.GetResult, error) + GetExisting(context.Context, []string) (*canvas.GetResult, error) + Apply(context.Context, canvas.ApplyOptions) (*canvas.ApplyResult, error) +} + +type runnerCanvasAPI struct { + runner *common.Runner +} + +func (api runnerCanvasAPI) Create(ctx context.Context, opts canvas.CreateOptions) (*canvas.CreateResult, error) { + return canvas.Create(ctx, opts, api.runner) +} + +func (api runnerCanvasAPI) ResumeCreate( + ctx context.Context, + accepted *canvas.CreateResult, + opts canvas.ResumeCreateOptions, +) (*canvas.CreateResult, error) { + return canvas.ResumeCreate(ctx, accepted, opts, api.runner) +} + +func (api runnerCanvasAPI) Allocate(ctx context.Context, count int) (*canvas.AllocateResult, error) { + return canvas.Allocate(ctx, count, api.runner) +} + +func (api runnerCanvasAPI) Get(ctx context.Context, assetIDs []string) (*canvas.GetResult, error) { + return canvas.Get(ctx, canvas.GetOptions{AssetIDs: assetIDs}, api.runner) +} + +func (api runnerCanvasAPI) GetExisting(ctx context.Context, assetIDs []string) (*canvas.GetResult, error) { + return canvas.GetExisting(ctx, assetIDs, api.runner) +} + +func (api runnerCanvasAPI) Apply(ctx context.Context, opts canvas.ApplyOptions) (*canvas.ApplyResult, error) { + return canvas.Apply(ctx, opts, api.runner) +} + +type Executor struct { + api canvasAPI +} + +func NewExecutor(runner *common.Runner) *Executor { + return &Executor{api: runnerCanvasAPI{runner: runner}} +} + +// Execute materializes a provider-neutral CanvasPlan into a personal novel +// canvas. An existing journal is resumed automatically and is never replayed +// after an ambiguous create or apply request. +func Execute( + ctx context.Context, + plan Plan, + resolved ResolvedMediaSet, + opts ExecuteOptions, + runner *common.Runner, +) (*ExecutionResult, error) { + if runner == nil || runner.Client == nil { + return nil, fmt.Errorf("CanvasPlan runner client is missing") + } + return NewExecutor(runner).Execute(ctx, plan, resolved, opts) +} + +// Resume is the existing-journal-only form of Execute. It fails without +// creating a journal when the requested operation has not been started. +func Resume( + ctx context.Context, + plan Plan, + resolved ResolvedMediaSet, + opts ExecuteOptions, + runner *common.Runner, +) (*ExecutionResult, error) { + if runner == nil || runner.Client == nil { + return nil, fmt.Errorf("CanvasPlan runner client is missing") + } + return NewExecutor(runner).Resume(ctx, plan, resolved, opts) +} diff --git a/internal/canvasplan/executor.go b/internal/canvasplan/executor.go new file mode 100644 index 0000000..ba04877 --- /dev/null +++ b/internal/canvasplan/executor.go @@ -0,0 +1,570 @@ +package canvasplan + +import ( + "context" + "fmt" + "path/filepath" + "regexp" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/canvas" +) + +var journalURLPattern = regexp.MustCompile(`https?://[^\s"']+`) + +func (executor *Executor) Execute( + ctx context.Context, + inputPlan Plan, + inputResolved ResolvedMediaSet, + opts ExecuteOptions, +) (*ExecutionResult, error) { + return executor.execute(ctx, inputPlan, inputResolved, opts, true) +} + +func (executor *Executor) Resume( + ctx context.Context, + inputPlan Plan, + inputResolved ResolvedMediaSet, + opts ExecuteOptions, +) (*ExecutionResult, error) { + return executor.execute(ctx, inputPlan, inputResolved, opts, false) +} + +func (executor *Executor) execute( + ctx context.Context, + inputPlan Plan, + inputResolved ResolvedMediaSet, + opts ExecuteOptions, + allowCreateJournal bool, +) (result *ExecutionResult, returnErr error) { + if executor == nil || executor.api == nil { + return nil, fmt.Errorf("CanvasPlan executor API is missing") + } + plan, err := NormalizePlan(inputPlan) + if err != nil { + return nil, err + } + resolved, err := NormalizeResolvedMedia(inputResolved) + if err != nil { + return nil, err + } + if err := ValidateResolution(plan, resolved); err != nil { + return nil, err + } + if len(plan.Nodes) > canvas.MaxAllocateCount { + return nil, fmt.Errorf("CanvasPlan contains %d business nodes; maximum is %d", len(plan.Nodes), canvas.MaxAllocateCount) + } + journalPath := strings.TrimSpace(opts.JournalPath) + if journalPath == "" { + return nil, fmt.Errorf("CanvasPlan journal path is required") + } + journalPath, err = filepath.Abs(journalPath) + if err != nil { + return nil, fmt.Errorf("resolve CanvasPlan journal path: %w", err) + } + journalPath = filepath.Clean(journalPath) + planHash, err := hashJSON(plan) + if err != nil { + return nil, fmt.Errorf("hash CanvasPlan: %w", err) + } + resolvedHash, err := hashJSON(resolved) + if err != nil { + return nil, fmt.Errorf("hash resolved media: %w", err) + } + + lock, err := acquireJournalLock(journalPath) + if err != nil { + return nil, err + } + defer func() { + if releaseErr := lock.release(); releaseErr != nil { + if returnErr == nil { + returnErr = fmt.Errorf("release CanvasPlan journal lock: %w", releaseErr) + } else { + returnErr = fmt.Errorf("%w; release CanvasPlan journal lock: %v", returnErr, releaseErr) + } + } + }() + + journal, _, err := loadOrCreateJournal(journalPath, planHash, resolvedHash, allowCreateJournal) + if err != nil { + return nil, err + } + result = executionResult(journalPath, journal, plan) + + if err := executor.ensureRoot(ctx, journalPath, journal, plan, opts); err != nil { + return executionResult(journalPath, journal, plan), err + } + if journal.State == StateCreatePending { + return executionResult(journalPath, journal, plan), nil + } + if err := executor.ensureAllocation(ctx, journalPath, journal, plan); err != nil { + return executionResult(journalPath, journal, plan), err + } + + document, err := Materialize(plan, resolved, journal.Create.CanvasAssetID, journal.NodeAssetIDs) + if err != nil { + return failExecution(journalPath, journal, plan, StateMaterializationDrift, err) + } + documentHash, err := DocumentSHA256(document) + if err != nil { + return failExecution(journalPath, journal, plan, StateMaterializationDrift, err) + } + assetHashes, err := DocumentAssetSHA256(document) + if err != nil { + return failExecution(journalPath, journal, plan, StateMaterializationDrift, err) + } + if journal.DocumentSHA256 != "" && journal.DocumentSHA256 != documentHash { + driftErr := fmt.Errorf("CanvasPlan materialization changed after it was journaled; refusing to apply") + return failExecution(journalPath, journal, plan, StateMaterializationDrift, driftErr) + } + if len(journal.AssetSHA256) != 0 && !equalStringMaps(journal.AssetSHA256, assetHashes) { + driftErr := fmt.Errorf("CanvasPlan asset materialization changed after it was journaled; refusing to apply") + return failExecution(journalPath, journal, plan, StateMaterializationDrift, driftErr) + } + if journal.DocumentSHA256 == "" { + journal.DocumentSHA256 = documentHash + journal.AssetSHA256 = assetHashes + journal.State = StateMaterialized + journal.LastError = "" + if err := saveJournal(journalPath, journal); err != nil { + return executionResult(journalPath, journal, plan), err + } + } + if journal.State == StateVerified { + if journal.Verification == nil || !journal.Verification.Verified { + return executionResult(journalPath, journal, plan), fmt.Errorf("verified CanvasPlan journal is missing successful verification") + } + return executor.reverifyCompleted(ctx, journalPath, journal, plan, document) + } + + return executor.applyAndVerify(ctx, journalPath, journal, plan, document) +} + +func (executor *Executor) reverifyCompleted( + ctx context.Context, + journalPath string, + journal *Journal, + plan Plan, + document *Document, +) (*ExecutionResult, error) { + assetIDs := DocumentAssetIDs(document) + queried, err := executor.api.Get(ctx, assetIDs) + if err != nil { + journal.Verification = &Verification{ExpectedAssetCount: len(assetIDs), Verified: false} + return failExecution( + journalPath, + journal, + plan, + StateVerificationFailed, + fmt.Errorf("verify completed CanvasPlan with current authentication: %w", err), + ) + } + verification := VerifyDocument(document, queried.Assets) + verification.LogID = queried.LogID + journal.Verification = &verification + if !verification.Verified { + return failExecution( + journalPath, + journal, + plan, + StateVerificationFailed, + fmt.Errorf( + "completed CanvasPlan no longer matches current authenticated assets: missing=%d unverifiable=%d mismatched=%d", + len(verification.MissingAssetIDs), + len(verification.UnverifiableAssetIDs), + len(verification.MismatchedAssetIDs), + ), + ) + } + journal.State = StateVerified + journal.LastError = "" + if err := saveJournal(journalPath, journal); err != nil { + return executionResult(journalPath, journal, plan), err + } + return executionResult(journalPath, journal, plan), nil +} + +func (executor *Executor) ensureRoot( + ctx context.Context, + journalPath string, + journal *Journal, + plan Plan, + opts ExecuteOptions, +) error { + switch journal.State { + case StateInitialized: + journal.State = StateCreateRequested + journal.LastError = "" + if err := saveJournal(journalPath, journal); err != nil { + return err + } + created, err := executor.api.Create(ctx, canvas.CreateOptions{ + Title: plan.Title, + RequestID: journal.RequestID, + Wait: true, + PollInterval: opts.PollInterval, + WaitTimeout: opts.WaitTimeout, + }) + return finishCreate(journalPath, journal, created, err) + case StateCreateRequested: + return recordJournalError( + journalPath, + journal, + StateCreateAmbiguous, + fmt.Errorf("canvas create may have been sent before interruption; do not create again blindly"), + ) + case StateCreatePending: + if journal.Create == nil { + return recordJournalError(journalPath, journal, StateCreateAmbiguous, fmt.Errorf("pending canvas create has no accepted IDs")) + } + created, err := executor.api.ResumeCreate(ctx, journal.Create, canvas.ResumeCreateOptions{ + PollInterval: opts.PollInterval, + WaitTimeout: opts.WaitTimeout, + }) + return finishCreate(journalPath, journal, created, err) + case StateCreateAmbiguous, StateCreateFailed: + return fmt.Errorf("CanvasPlan create is in terminal safety state %q; inspect journal and accepted IDs before retrying", journal.State) + default: + if journal.Create == nil || journal.Create.State != canvas.StateReady { + return fmt.Errorf("CanvasPlan journal state %q has no ready personal novel canvas", journal.State) + } + return nil + } +} + +func finishCreate(journalPath string, journal *Journal, created *canvas.CreateResult, createErr error) error { + if created != nil { + journal.Create = created + } + if createErr != nil { + state := StateCreateAmbiguous + if created != nil { + state = StateCreateFailed + } + return recordJournalError(journalPath, journal, state, createErr) + } + if created == nil { + return recordJournalError(journalPath, journal, StateCreateAmbiguous, fmt.Errorf("canvas create returned no result; do not create again blindly")) + } + if created.State != canvas.StateReady { + journal.State = StateCreatePending + journal.LastError = sanitizeJournalError(created.Warning) + return saveJournal(journalPath, journal) + } + journal.State = StateRootReady + journal.LastError = "" + return saveJournal(journalPath, journal) +} + +func (executor *Executor) ensureAllocation(ctx context.Context, journalPath string, journal *Journal, plan Plan) error { + if len(journal.NodeAssetIDs) == len(plan.Nodes) { + return nil + } + if len(journal.NodeAssetIDs) != 0 { + return recordJournalError( + journalPath, + journal, + StateMaterializationDrift, + fmt.Errorf("CanvasPlan journal has %d allocated node IDs, want %d", len(journal.NodeAssetIDs), len(plan.Nodes)), + ) + } + if journal.State != StateRootReady && journal.State != StateAllocationRequested { + return fmt.Errorf("CanvasPlan journal state %q is missing allocated node IDs", journal.State) + } + if journal.State == StateRootReady { + journal.State = StateAllocationRequested + journal.LastError = "" + if err := saveJournal(journalPath, journal); err != nil { + return err + } + } + allocation, err := executor.api.Allocate(ctx, len(plan.Nodes)) + if err != nil { + return recordJournalError(journalPath, journal, StateAllocationRequested, err) + } + if allocation == nil || len(allocation.AssetIDs) != len(plan.Nodes) { + return recordJournalError(journalPath, journal, StateAllocationRequested, fmt.Errorf("canvas allocation result is incomplete")) + } + journal.NodeAssetIDs = make(map[string]string, len(plan.Nodes)) + for index, node := range plan.Nodes { + journal.NodeAssetIDs[node.LogicalID] = allocation.AssetIDs[index] + } + journal.AllocationLogID = allocation.LogID + journal.State = StateAllocated + journal.LastError = "" + return saveJournal(journalPath, journal) +} + +func (executor *Executor) applyAndVerify( + ctx context.Context, + journalPath string, + journal *Journal, + plan Plan, + document *Document, +) (*ExecutionResult, error) { + assetIDs := DocumentAssetIDs(document) + existing, err := executor.api.GetExisting(ctx, assetIDs) + if err != nil { + return failExecution(journalPath, journal, plan, journal.State, fmt.Errorf("query CanvasPlan assets before apply: %w", err)) + } + indexed, err := indexQueriedAssets(existing.Assets) + if err != nil { + return failExecution(journalPath, journal, plan, journal.State, fmt.Errorf("index CanvasPlan preflight assets: %w", err)) + } + verification := VerifyDocument(document, existing.Assets) + verification.LogID = existing.LogID + if verification.Verified { + verification.RecoveredFromQuery = true + journal.Verification = &verification + journal.State = StateVerified + journal.LastError = "" + if journal.Apply != nil { + journal.Apply.Status = "verified" + } + if err := saveJournal(journalPath, journal); err != nil { + return executionResult(journalPath, journal, plan), err + } + return executionResult(journalPath, journal, plan), nil + } + if companionIDs := existingCompanionIDs(document, indexed); len(companionIDs) != 0 { + partialErr := fmt.Errorf("query found %d companion assets without an exact full document match; refusing partial apply replay", len(companionIDs)) + journal.Verification = &verification + return failExecution(journalPath, journal, plan, StateUnsafePartial, partialErr) + } + rootRaw, rootExists := indexed[document.RootCanvasID] + if !rootExists { + missingErr := fmt.Errorf("created Canvas root %q is not queryable", document.RootCanvasID) + if journal.Apply != nil && journal.Apply.Status != "prepared" { + journal.Verification = &verification + return failExecution(journalPath, journal, plan, StateVerificationFailed, missingErr) + } + return failExecution(journalPath, journal, plan, journal.State, missingErr) + } + rootVersion, err := queriedAssetVersion(rootRaw) + if err != nil { + return failExecution(journalPath, journal, plan, journal.State, fmt.Errorf("read created Canvas root version: %w", err)) + } + + request, err := prepareApplyRequest(journal, document, rootVersion) + if err != nil { + state := journal.State + if strings.Contains(err.Error(), "root changed") { + state = StateUnsafeRootChanged + } + return failExecution(journalPath, journal, plan, state, err) + } + if journal.Apply != nil && journal.Apply.Status != "prepared" { + return executionResult(journalPath, journal, plan), fmt.Errorf( + "CanvasPlan apply is journaled as %q but query-back did not verify; refusing to replay blindly", + journal.Apply.Status, + ) + } + if journal.Apply == nil { + requestHash, hashErr := hashJSON(request) + if hashErr != nil { + return executionResult(journalPath, journal, plan), hashErr + } + journal.Apply = &ApplyJournal{ + TransactionID: request.Transactions[0].TransactionID, + BatchID: request.BatchID, + ClientID: request.ClientID, + BaseRootVersion: rootVersion, + RequestSHA256: requestHash, + Status: "prepared", + } + journal.State = StateApplyPrepared + journal.LastError = "" + if err := saveJournal(journalPath, journal); err != nil { + return executionResult(journalPath, journal, plan), err + } + } + + journal.State = StateApplyRequested + journal.Apply.Status = "requested" + journal.LastError = "" + if err := saveJournal(journalPath, journal); err != nil { + return executionResult(journalPath, journal, plan), err + } + applyResult, err := executor.api.Apply(ctx, canvas.ApplyOptions{ProjectID: journal.Create.ProjectID, Request: request}) + if err != nil { + journal.Apply.Status = "ambiguous" + return failExecution(journalPath, journal, plan, StateApplyAmbiguous, err) + } + if applyResult == nil || len(applyResult.Results) != 1 { + journal.Apply.Status = "ambiguous" + return failExecution( + journalPath, + journal, + plan, + StateApplyAmbiguous, + fmt.Errorf("canvas apply acknowledgement is incomplete; query affected assets and do not replay blindly"), + ) + } + journal.Apply.Status = "acknowledged" + journal.Apply.AssetVersions = applyResult.Results[0].AssetVersions + journal.Apply.LogID = applyResult.LogID + journal.State = StateApplyAcknowledged + journal.LastError = "" + if err := saveJournal(journalPath, journal); err != nil { + return executionResult(journalPath, journal, plan), err + } + + queried, err := executor.api.Get(ctx, assetIDs) + if err != nil { + journal.Apply.Status = "verification-failed" + return failExecution(journalPath, journal, plan, StateVerificationFailed, fmt.Errorf("query CanvasPlan assets after apply: %w", err)) + } + verification = VerifyDocument(document, queried.Assets) + verification.LogID = queried.LogID + journal.Verification = &verification + if !verification.Verified { + journal.Apply.Status = "verification-failed" + verifyErr := fmt.Errorf( + "CanvasPlan query-back verification failed: missing=%d unverifiable=%d mismatched=%d; do not replay blindly", + len(verification.MissingAssetIDs), + len(verification.UnverifiableAssetIDs), + len(verification.MismatchedAssetIDs), + ) + return failExecution(journalPath, journal, plan, StateVerificationFailed, verifyErr) + } + journal.Apply.Status = "verified" + journal.State = StateVerified + journal.LastError = "" + if err := saveJournal(journalPath, journal); err != nil { + return executionResult(journalPath, journal, plan), err + } + return executionResult(journalPath, journal, plan), nil +} + +func prepareApplyRequest(journal *Journal, document *Document, rootVersion int64) (canvas.ApplyRequest, error) { + if journal.Create == nil || !isPositiveDecimal(journal.Create.ProjectID) { + return canvas.ApplyRequest{}, fmt.Errorf("created personal novel project_id must be a positive decimal JSON string") + } + stable := hashBytes([]byte(journal.OperationID + ":" + journal.DocumentSHA256))[:24] + transactionID := "canvas_plan_" + stable + request := canvas.ApplyRequest{ + BatchID: "batch_" + transactionID, + ClientID: "pippit_cli_canvas_plan_" + stable, + RootPippitAssetID: document.RootCanvasID, + Base: map[string]any{}, + Transactions: []canvas.PatchTransaction{{ + TransactionID: transactionID, + Intent: "canvas.write", + }}, + } + rootVersionCopy := rootVersion + request.Transactions[0].Patches = append(request.Transactions[0].Patches, canvas.PatchEntry{ + AssetID: document.RootCanvasID, + BaseAssetVersion: &rootVersionCopy, + Op: "replace", + Path: "", + Value: document.Assets[document.RootCanvasID], + }) + for _, assetID := range DocumentAssetIDs(document)[1:] { + zero := int64(0) + request.Transactions[0].Patches = append(request.Transactions[0].Patches, canvas.PatchEntry{ + AssetID: assetID, + BaseAssetVersion: &zero, + Op: "add", + Path: "", + Value: document.Assets[assetID], + }) + } + requestHash, err := hashJSON(request) + if err != nil { + return canvas.ApplyRequest{}, err + } + if journal.Apply != nil { + if journal.Apply.BaseRootVersion != rootVersion { + return canvas.ApplyRequest{}, fmt.Errorf("created Canvas root changed after apply was prepared; refusing to overwrite it") + } + if journal.Apply.TransactionID != transactionID || journal.Apply.BatchID != request.BatchID || + journal.Apply.ClientID != request.ClientID || journal.Apply.RequestSHA256 != requestHash { + return canvas.ApplyRequest{}, fmt.Errorf("journaled CanvasPlan apply request differs from materialized request; refusing to replay") + } + } + return request, nil +} + +func executionResult(journalPath string, journal *Journal, plan Plan) *ExecutionResult { + if journal == nil { + return nil + } + result := &ExecutionResult{ + State: journal.State, + JournalPath: journalPath, + OperationID: journal.OperationID, + DocumentSHA256: journal.DocumentSHA256, + NodeCount: len(plan.Nodes) + len(plan.Groups), + EdgeCount: len(plan.Edges), + Verification: journal.Verification, + } + if journal.DocumentSHA256 != "" { + result.AssetCount = len(journal.AssetSHA256) + } + if journal.Create != nil { + result.ProjectID = journal.Create.ProjectID + result.RootCanvasID = journal.Create.CanvasAssetID + result.OverviewPippitAssetID = journal.Create.OverviewPippitAssetID + result.WebURL = journal.Create.WebURL + if journal.State == StateCreatePending { + result.Warning = journal.Create.Warning + } + } + if journal.Apply != nil { + result.TransactionID = journal.Apply.TransactionID + } + if result.Warning == "" && strings.Contains(journal.State, "ambiguous") { + result.Warning = journal.LastError + } + return result +} + +func recordJournalError(journalPath string, journal *Journal, state string, cause error) error { + journal.State = state + journal.LastError = sanitizeJournalError(cause.Error()) + if err := saveJournal(journalPath, journal); err != nil { + return fmt.Errorf("%w; additionally failed to save CanvasPlan journal: %v", cause, err) + } + return cause +} + +func failExecution(journalPath string, journal *Journal, plan Plan, state string, cause error) (*ExecutionResult, error) { + err := recordJournalError(journalPath, journal, state, cause) + return executionResult(journalPath, journal, plan), err +} + +func sanitizeJournalError(value string) string { + value = journalURLPattern.ReplaceAllString(strings.TrimSpace(value), "[redacted-url]") + const limit = 1024 + if len(value) > limit { + return value[:limit] + "..." + } + return value +} + +func equalStringMaps(left, right map[string]string) bool { + if len(left) != len(right) { + return false + } + for key, value := range left { + if right[key] != value { + return false + } + } + return true +} + +func isPositiveDecimal(value string) bool { + value = strings.TrimSpace(value) + if value == "" || value[0] == '0' { + return false + } + for _, char := range value { + if char < '0' || char > '9' { + return false + } + } + return true +} From ac5bdf65cfd3a5554a28885dd987c494258f1f5c Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:46:50 +0800 Subject: [PATCH 11/48] test: cover CanvasPlan recovery boundaries Co-authored-by: Codex <codex@openai.com> --- internal/canvasplan/canvasplan_test.go | 642 ++++++++++++++++++ .../canvasplan/journal_security_unix_test.go | 139 ++++ 2 files changed, 781 insertions(+) create mode 100644 internal/canvasplan/canvasplan_test.go create mode 100644 internal/canvasplan/journal_security_unix_test.go diff --git a/internal/canvasplan/canvasplan_test.go b/internal/canvasplan/canvasplan_test.go new file mode 100644 index 0000000..1e142a7 --- /dev/null +++ b/internal/canvasplan/canvasplan_test.go @@ -0,0 +1,642 @@ +package canvasplan + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Pippit-dev/pippit-cli/internal/canvas" +) + +func TestMaterializeCanonicalDocumentWithoutTransientMediaLocations(t *testing.T) { + plan, resolved := testPlanAndResolved() + mapping := map[string]string{ + "node:image": "node-asset-image", + "node:video": "node-asset-video", + "node:audio": "node-asset-audio", + "node:composite": "node-asset-composite", + "node:image-placeholder": "node-asset-image-placeholder", + "node:video-placeholder": "node-asset-video-placeholder", + } + document, err := Materialize(plan, resolved, "root-asset", mapping) + if err != nil { + t.Fatalf("Materialize() error = %v", err) + } + if got, want := len(document.Assets), len(plan.Nodes)+1; got != want { + t.Fatalf("asset count = %d, want %d", got, want) + } + payload, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"https://source.example", "bundle/media"} { + if bytes.Contains(payload, []byte(forbidden)) { + t.Fatalf("materialized document persisted transient source %q: %s", forbidden, payload) + } + } + + root := decodeRawMap(t, document.Assets[document.RootCanvasID]) + content := asMap(t, root["content"]) + nodes := asMap(t, content["nodes"]) + group := asMap(t, nodes["group:scene"]) + children := asSlice(t, group["children"]) + if got := fmt.Sprint(children[0]); got != mapping["node:image"] { + t.Fatalf("group first child = %q, want allocated ID", got) + } + edges := asMap(t, content["edges"]) + edge := asMap(t, edges["edge:reference"]) + if edge["source"] != mapping["node:video"] || edge["target"] != mapping["node:composite"] { + t.Fatalf("edge endpoints were not remapped: %#v", edge) + } + + image := companionContent(t, document, mapping["node:image"]) + if image["assetId"] != "upload-image" || image["pippitAssetId"] != "media-image" || image["sourceType"] != float64(4) { + t.Fatalf("image content = %#v", image) + } + if image["naturalWidth"] != float64(1600) || image["naturalHeight"] != float64(900) { + t.Fatalf("image dimensions = %#v", image) + } + video := companionContent(t, document, mapping["node:video"]) + if video["duration"] != float64(6) || video["frameWidth"] != float64(1280) || video["frameHeight"] != float64(720) { + t.Fatalf("video content = %#v", video) + } + if _, invented := video["generation"]; invented { + t.Fatalf("uploaded video content unexpectedly contains generation: %#v", video) + } + composite := companionContent(t, document, mapping["node:composite"]) + references := asSlice(t, asMap(t, composite["generation"])["references"]) + if reference := asMap(t, references[0]); reference["nodeAssetId"] != mapping["node:video"] { + t.Fatalf("composite reference = %#v", reference) + } + imagePlaceholder := companionContent(t, document, mapping["node:image-placeholder"]) + if _, hasAssetID := imagePlaceholder["assetId"]; hasAssetID { + t.Fatalf("placeholder persisted an asset ID: %#v", imagePlaceholder) + } +} + +func TestDecodeContractsAreStrictAndAllowDeduplicatedResolvedAssets(t *testing.T) { + plan, resolved := testPlanAndResolved() + resolved.Media[1].AssetID = resolved.Media[0].AssetID + resolved.Media[1].PippitAssetID = resolved.Media[0].PippitAssetID + if _, err := NormalizeResolvedMedia(resolved); err != nil { + t.Fatalf("NormalizeResolvedMedia() rejected deduplicated upload: %v", err) + } + + planPayload, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + planPayload = bytes.Replace(planPayload, []byte(`"title":`), []byte(`"team_id":"forbidden","title":`), 1) + if _, err := DecodePlan(bytes.NewReader(planPayload)); err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("DecodePlan() error = %v, want unknown team_id rejection", err) + } + + resolvedPayload, err := json.Marshal(resolved) + if err != nil { + t.Fatal(err) + } + resolvedPayload = bytes.Replace(resolvedPayload, []byte(`"schema":`), []byte(`"team_id":"forbidden","schema":`), 1) + if _, err := DecodeResolvedMedia(bytes.NewReader(resolvedPayload)); err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("DecodeResolvedMedia() error = %v, want unknown team_id rejection", err) + } + + badSource := plan + badSource.RequiredMedia = append([]MediaRequirement(nil), plan.RequiredMedia...) + badSource.RequiredMedia[0].URL = "https://source.example/image.png" + if _, err := NormalizePlan(badSource); err == nil || !strings.Contains(err.Error(), "exactly one") { + t.Fatalf("NormalizePlan() error = %v, want URL/local_path XOR rejection", err) + } + badDigest := plan + badDigest.RequiredMedia = append([]MediaRequirement(nil), plan.RequiredMedia...) + badDigest.RequiredMedia[0].SHA256 = "sha256:bad" + if _, err := NormalizePlan(badDigest); err == nil || !strings.Contains(err.Error(), "64 lowercase") { + t.Fatalf("NormalizePlan() error = %v, want digest rejection", err) + } +} + +func TestHashRawJSONPreservesIntegerPrecision(t *testing.T) { + left, err := hashRawJSON(json.RawMessage(`{"value":9007199254740992}`)) + if err != nil { + t.Fatal(err) + } + right, err := hashRawJSON(json.RawMessage(`{"value":9007199254740993}`)) + if err != nil { + t.Fatal(err) + } + if left == right { + t.Fatal("hashRawJSON() collapsed distinct large integers") + } +} + +func TestExternalPlanFixtureWhenConfigured(t *testing.T) { + planPath := strings.TrimSpace(os.Getenv("PIPPIT_CANVASPLAN_FIXTURE")) + if planPath == "" { + t.Skip("set PIPPIT_CANVASPLAN_FIXTURE to run a local exported-plan smoke test") + } + file, err := os.Open(planPath) + if err != nil { + t.Fatal(err) + } + plan, err := DecodePlan(file) + _ = file.Close() + if err != nil { + t.Fatalf("DecodePlan(%s) error = %v", planPath, err) + } + resolved := ResolvedMediaSet{Schema: ResolvedMediaSchema, Media: make([]ResolvedMedia, 0, len(plan.RequiredMedia))} + for index, media := range plan.RequiredMedia { + localPath := filepath.Join(filepath.Dir(planPath), filepath.FromSlash(media.LocalPath)) + payload, err := os.ReadFile(localPath) + if err != nil { + t.Fatalf("read media %q: %v", media.LogicalID, err) + } + if media.Metadata.ByteSize == nil || int64(len(payload)) != *media.Metadata.ByteSize { + t.Fatalf("media %q byte size mismatch", media.LogicalID) + } + if got := fmt.Sprintf("%x", sha256.Sum256(payload)); got != media.SHA256 { + t.Fatalf("media %q SHA-256 mismatch", media.LogicalID) + } + resolved.Media = append(resolved.Media, ResolvedMedia{ + LogicalID: media.LogicalID, MediaType: media.MediaType, + AssetID: fmt.Sprintf("fixture-upload-%d", index+1), PippitAssetID: fmt.Sprintf("fixture-media-%d", index+1), + }) + } + mapping := make(map[string]string, len(plan.Nodes)) + for index, node := range plan.Nodes { + mapping[node.LogicalID] = fmt.Sprintf("fixture-node-%d", index+1) + } + document, err := Materialize(plan, resolved, "fixture-root", mapping) + if err != nil { + t.Fatalf("Materialize(real plan) error = %v", err) + } + if got, want := len(document.Assets), len(plan.Nodes)+1; got != want { + t.Fatalf("materialized asset count = %d, want %d", got, want) + } + t.Logf( + "materialized title=%q media=%d nodes=%d groups=%d edges=%d degradations=%d assets=%d", + plan.Title, len(plan.RequiredMedia), len(plan.Nodes), len(plan.Groups), len(plan.Edges), len(plan.Degradations), len(document.Assets), + ) +} + +func TestExecutorAppliesOnceVerifiesAndReusesJournal(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "canvas-plan.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil { + t.Fatalf("Execute() error = %v", err) + } + if result.State != StateVerified || result.Verification == nil || !result.Verification.Verified { + t.Fatalf("Execute() result = %#v, want verified", result) + } + if api.applyCalls != 1 || api.createCalls != 1 || api.allocateCalls != 1 { + t.Fatalf("remote calls create=%d allocate=%d apply=%d", api.createCalls, api.allocateCalls, api.applyCalls) + } + request := api.lastApply.Request + if len(request.Transactions) != 1 || request.Transactions[0].Intent != "canvas.write" { + t.Fatalf("apply request = %#v, want one canvas.write transaction", request) + } + if got, want := len(request.Transactions[0].Patches), len(plan.Nodes)+1; got != want { + t.Fatalf("patch count = %d, want %d", got, want) + } + if request.Transactions[0].Patches[0].Op != "replace" || request.Transactions[0].Patches[0].BaseAssetVersion == nil || *request.Transactions[0].Patches[0].BaseAssetVersion != 7 { + t.Fatalf("root patch = %#v", request.Transactions[0].Patches[0]) + } + info, err := os.Stat(journalPath) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("journal mode = %o, want 600", got) + } + + callsBefore := api.totalCalls() + getCallsBefore := api.getCalls + resumed, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil { + t.Fatalf("second Execute() error = %v", err) + } + if resumed.State != StateVerified || api.totalCalls() != callsBefore+1 || api.getCalls != getCallsBefore+1 { + t.Fatalf("second Execute() result=%#v total calls=%d get calls=%d, want one current-auth query", resumed, api.totalCalls()-callsBefore, api.getCalls-getCallsBefore) + } + if api.applyCalls != 1 { + t.Fatalf("verified resume replayed apply: calls=%d", api.applyCalls) + } +} + +func TestExecutorVerifiedResumeFailsOnCurrentAccountMismatch(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "current-account-mismatch.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil || result.State != StateVerified { + t.Fatalf("first Execute() result=%#v error=%v, want verified", result, err) + } + api.getErr = errors.New("assets are not visible to current account") + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err == nil || result == nil || result.State != StateVerificationFailed { + t.Fatalf("verified resume result=%#v error=%v, want current-auth verification failure", result, err) + } + if result.Verification == nil || result.Verification.Verified { + t.Fatalf("verification = %#v, want fresh failed verification", result.Verification) + } + if api.applyCalls != 1 { + t.Fatalf("verified resume replayed apply: calls=%d", api.applyCalls) + } +} + +func TestExecutorVerifiedResumeFailsOnRemoteDrift(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "remote-drift.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil || result.State != StateVerified { + t.Fatalf("first Execute() result=%#v error=%v, want verified", result, err) + } + api.corruptGet = true + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err == nil || result == nil || result.State != StateVerificationFailed { + t.Fatalf("verified resume result=%#v error=%v, want remote-drift failure", result, err) + } + if result.Verification == nil || result.Verification.Verified || len(result.Verification.MismatchedAssetIDs) != 1 { + t.Fatalf("verification = %#v, want one mismatched asset", result.Verification) + } + if api.applyCalls != 1 { + t.Fatalf("verified resume replayed apply: calls=%d", api.applyCalls) + } +} + +func TestExecutorNeverReplaysAmbiguousApply(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + api.applyErr = errors.New("connection lost after request") + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "ambiguous.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err == nil || result == nil || result.State != StateApplyAmbiguous { + t.Fatalf("Execute() result=%#v error=%v, want apply ambiguity", result, err) + } + if api.applyCalls != 1 { + t.Fatalf("apply calls = %d, want 1", api.applyCalls) + } + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err == nil || result == nil || !strings.Contains(err.Error(), "refusing to replay") { + t.Fatalf("resume result=%#v error=%v, want replay refusal", result, err) + } + if api.applyCalls != 1 { + t.Fatalf("apply was replayed: calls=%d", api.applyCalls) + } + journal := readJournal(t, journalPath) + if journal.Apply == nil || journal.Apply.Status != "ambiguous" { + t.Fatalf("journal apply = %#v", journal.Apply) + } +} + +func TestExecutorRecoversCommittedAmbiguousApplyByQuery(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + api.applyErr = errors.New("connection lost after commit") + api.commitBeforeApplyError = true + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "committed-ambiguous.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err == nil || result.State != StateApplyAmbiguous { + t.Fatalf("first Execute() result=%#v error=%v, want ambiguity", result, err) + } + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil || result.State != StateVerified || result.Verification == nil || !result.Verification.RecoveredFromQuery { + t.Fatalf("resume result=%#v error=%v, want query recovery", result, err) + } + if api.applyCalls != 1 { + t.Fatalf("committed apply was replayed: calls=%d", api.applyCalls) + } +} + +func TestExecutorNeverReplaysAmbiguousCreate(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + api.createErr = errors.New("connection lost after create request") + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "ambiguous-create.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err == nil || result == nil || result.State != StateCreateAmbiguous { + t.Fatalf("Execute() result=%#v error=%v, want create ambiguity", result, err) + } + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err == nil || result == nil || !strings.Contains(err.Error(), "terminal safety state") { + t.Fatalf("resume result=%#v error=%v, want create replay refusal", result, err) + } + if api.createCalls != 1 { + t.Fatalf("create was replayed: calls=%d", api.createCalls) + } +} + +func TestExecutorReportsQueryBackMismatchWithoutReplay(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + api.corruptGet = true + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "query-mismatch.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err == nil || result == nil || result.State != StateVerificationFailed { + t.Fatalf("Execute() result=%#v error=%v, want verification failure", result, err) + } + if result.Verification == nil || len(result.Verification.MismatchedAssetIDs) != 1 { + t.Fatalf("verification = %#v", result.Verification) + } + if api.applyCalls != 1 { + t.Fatalf("apply calls = %d, want 1", api.applyCalls) + } +} + +func TestExecutorResumesAcceptedCreateWithoutCreatingAgain(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + api.createPending = true + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "pending-create.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil || result.State != StateCreatePending || result.ProjectID == "" { + t.Fatalf("first Execute() result=%#v error=%v", result, err) + } + if api.createCalls != 1 || api.resumeCreateCalls != 0 { + t.Fatalf("first call create=%d resume=%d", api.createCalls, api.resumeCreateCalls) + } + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil || result.State != StateVerified { + t.Fatalf("second Execute() result=%#v error=%v", result, err) + } + if api.createCalls != 1 || api.resumeCreateCalls != 1 { + t.Fatalf("create was duplicated: create=%d resume=%d", api.createCalls, api.resumeCreateCalls) + } +} + +func TestExecutorRejectsJournalInputDriftAndResumeWithoutJournal(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + api.createPending = true + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "drift.json") + if _, err := executor.Resume(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}); err == nil || !strings.Contains(err.Error(), "does not exist") { + t.Fatalf("Resume() error = %v, want missing journal rejection", err) + } + if _, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}); err != nil { + t.Fatal(err) + } + changed := plan + changed.Title = "Changed title" + if _, err := executor.Execute(context.Background(), changed, resolved, ExecuteOptions{JournalPath: journalPath}); err == nil || !strings.Contains(err.Error(), "changed") { + t.Fatalf("Execute(changed plan) error = %v, want drift rejection", err) + } +} + +func testPlanAndResolved() (Plan, ResolvedMediaSet) { + byteSizeImage, byteSizeVideo, byteSizeAudio := int64(1234), int64(5678), int64(234) + durationVideo, durationAudio := int64(6000), int64(9000) + imageWidth, imageHeight := int64(1600), int64(900) + videoWidth, videoHeight := int64(1280), int64(720) + plan := Plan{ + Schema: PlanSchema, + Title: "Provider-neutral import", + Source: Source{Provider: "fixture", ProjectID: "source-project", Fingerprint: "fingerprint-1"}, + RequiredMedia: []MediaRequirement{ + { + LogicalID: "media:image", SourceNodeID: "source-image", FileName: "image.png", MediaType: "image", + LocalPath: "bundle/media/image.png", SHA256: strings.Repeat("a", 64), + Metadata: MediaMetadata{ByteSize: &byteSizeImage, Extension: "png", Height: &imageHeight, MimeType: "image/png", Width: &imageWidth}, + }, + { + LogicalID: "media:video", SourceNodeID: "source-video", FileName: "video.mp4", MediaType: "video", + LocalPath: "bundle/media/video.mp4", SHA256: strings.Repeat("b", 64), + Metadata: MediaMetadata{ByteSize: &byteSizeVideo, DurationMS: &durationVideo, Extension: "mp4", Height: &videoHeight, MimeType: "video/mp4", Width: &videoWidth}, + }, + { + LogicalID: "media:audio", SourceNodeID: "source-audio", FileName: "audio.mp3", MediaType: "audio", + URL: "https://source.example/audio.mp3", + Metadata: MediaMetadata{ByteSize: &byteSizeAudio, DurationMS: &durationAudio, Extension: "mp3", MimeType: "audio/mpeg"}, + }, + }, + Nodes: []Node{ + {LogicalID: "node:image", SourceNodeID: "source-image", Title: "Image", Position: Position{X: 10, Y: 20}, Size: Size{Width: 320, Height: 180}, ParentGroupLogicalID: "group:scene", Order: 0, Kind: "image", TargetType: "biz/image", MediaLogicalID: "media:image"}, + {LogicalID: "node:video", SourceNodeID: "source-video", Title: "Video", Position: Position{X: 350, Y: 20}, Size: Size{Width: 320, Height: 180}, ParentGroupLogicalID: "group:scene", Order: 1, Kind: "video", TargetType: "biz/video", MediaLogicalID: "media:video"}, + {LogicalID: "node:audio", SourceNodeID: "source-audio", Title: "Audio", Position: Position{X: 10, Y: 220}, Size: Size{Width: 320, Height: 100}, ParentGroupLogicalID: "group:scene", Order: 2, Kind: "audio", TargetType: "biz/audio", MediaLogicalID: "media:audio"}, + {LogicalID: "node:composite", SourceNodeID: "source-composite", Title: "Composite", Position: Position{X: 350, Y: 220}, Size: Size{Width: 320, Height: 180}, ParentGroupLogicalID: "group:scene", Order: 3, Kind: "video-composite", TargetType: "biz/video", Variant: "video-composite", InputNodeLogicalIDs: []string{"node:video"}}, + {LogicalID: "node:image-placeholder", SourceNodeID: "source-image-placeholder", Title: "Pending image", Position: Position{X: 10, Y: 420}, Size: Size{Width: 320, Height: 180}, ParentGroupLogicalID: "group:scene", Order: 4, Kind: "image-placeholder", TargetType: "biz/image"}, + {LogicalID: "node:video-placeholder", SourceNodeID: "source-video-placeholder", Title: "Pending video", Position: Position{X: 350, Y: 420}, Size: Size{Width: 320, Height: 180}, ParentGroupLogicalID: "group:scene", Order: 5, Kind: "video-placeholder", TargetType: "biz/video"}, + }, + Groups: []Group{{ + LogicalID: "group:scene", SourceNodeID: "source-group", Title: "Scene", Position: Position{X: 0, Y: 0}, Size: Size{Width: 700, Height: 640}, Order: 0, + ChildLogicalIDs: []string{"node:image", "node:video", "node:audio", "node:composite", "node:image-placeholder", "node:video-placeholder"}, + }}, + Edges: []Edge{{LogicalID: "edge:reference", SourceEdgeID: "source-edge", Type: "reference", SourceNodeLogicalID: "node:video", TargetNodeLogicalID: "node:composite", SourceHandle: "right", TargetHandle: "left"}}, + Degradations: []json.RawMessage{json.RawMessage(`{"reason":"source media is not ready","node_logical_ids":["node:image-placeholder","node:video-placeholder"]}`)}, + } + resolved := ResolvedMediaSet{ + Schema: ResolvedMediaSchema, + Media: []ResolvedMedia{ + {LogicalID: "media:image", MediaType: "image", AssetID: "upload-image", PippitAssetID: "media-image"}, + {LogicalID: "media:video", MediaType: "video", AssetID: "upload-video", PippitAssetID: "media-video"}, + {LogicalID: "media:audio", MediaType: "audio", AssetID: "upload-audio", PippitAssetID: "media-audio"}, + }, + } + return plan, resolved +} + +type fakeCanvasAPI struct { + nodeCount int + createCalls int + resumeCreateCalls int + allocateCalls int + getCalls int + getExistingCalls int + applyCalls int + createPending bool + createErr error + getErr error + applyErr error + commitBeforeApplyError bool + corruptGet bool + lastApply canvas.ApplyOptions + stored map[string]json.RawMessage +} + +func newFakeCanvasAPI(nodeCount int) *fakeCanvasAPI { + return &fakeCanvasAPI{nodeCount: nodeCount, stored: make(map[string]json.RawMessage)} +} + +func (api *fakeCanvasAPI) Create(context.Context, canvas.CreateOptions) (*canvas.CreateResult, error) { + api.createCalls++ + if api.createErr != nil { + return nil, api.createErr + } + state := canvas.StateReady + warning := "" + if api.createPending { + state = canvas.StateCreating + warning = "still creating" + } + return &canvas.CreateResult{ + RequestID: "request-1", State: state, ProjectID: "123", ThreadID: "thread-1", RunID: "run-1", + CanvasAssetID: "root-asset", OverviewPippitAssetID: "overview-asset", WebURL: "/novel/detail/canvas?projectId=123", Warning: warning, + }, nil +} + +func (api *fakeCanvasAPI) ResumeCreate(context.Context, *canvas.CreateResult, canvas.ResumeCreateOptions) (*canvas.CreateResult, error) { + api.resumeCreateCalls++ + api.createPending = false + return &canvas.CreateResult{ + RequestID: "request-1", State: canvas.StateReady, ProjectID: "123", ThreadID: "thread-1", RunID: "run-1", + CanvasAssetID: "root-asset", OverviewPippitAssetID: "overview-asset", WebURL: "/novel/detail/canvas?projectId=123", + }, nil +} + +func (api *fakeCanvasAPI) Allocate(_ context.Context, count int) (*canvas.AllocateResult, error) { + api.allocateCalls++ + if count != api.nodeCount { + return nil, fmt.Errorf("count=%d, want %d", count, api.nodeCount) + } + ids := make([]string, count) + for index := range ids { + ids[index] = fmt.Sprintf("node-asset-%d", index+1) + } + return &canvas.AllocateResult{AssetIDs: ids, LogID: "allocate-log"}, nil +} + +func (api *fakeCanvasAPI) Get(_ context.Context, assetIDs []string) (*canvas.GetResult, error) { + api.getCalls++ + if api.getErr != nil { + return nil, api.getErr + } + assets := make([]json.RawMessage, 0, len(assetIDs)) + for _, assetID := range assetIDs { + content, ok := api.stored[assetID] + if !ok { + return nil, fmt.Errorf("missing stored asset %s", assetID) + } + if api.corruptGet && assetID == "root-asset" { + content = json.RawMessage(`{"pippitAssetId":"root-asset","type":"canvas","content":{"corrupted":true},"extra":{}}`) + } + assets = append(assets, queriedAsset(testedAsset{ID: assetID, Version: 8, Content: content})) + } + return &canvas.GetResult{RequestedAssetIDs: assetIDs, Assets: assets, LogID: "query-after-log"}, nil +} + +func (api *fakeCanvasAPI) GetExisting(_ context.Context, assetIDs []string) (*canvas.GetResult, error) { + api.getExistingCalls++ + assets := make([]json.RawMessage, 0, len(assetIDs)) + for _, assetID := range assetIDs { + if content, ok := api.stored[assetID]; ok { + assets = append(assets, queriedAsset(testedAsset{ID: assetID, Version: 8, Content: content})) + continue + } + if assetID == "root-asset" { + assets = append(assets, queriedAsset(testedAsset{ID: assetID, Version: 7, Content: json.RawMessage(`{"pippitAssetId":"root-asset","type":"canvas","content":{},"extra":{}}`)})) + } + } + return &canvas.GetResult{RequestedAssetIDs: assetIDs, Assets: assets, LogID: "preflight-log"}, nil +} + +func (api *fakeCanvasAPI) Apply(_ context.Context, opts canvas.ApplyOptions) (*canvas.ApplyResult, error) { + api.applyCalls++ + api.lastApply = opts + if api.applyErr != nil && !api.commitBeforeApplyError { + return nil, api.applyErr + } + versions := make(map[string]int64) + for _, patch := range opts.Request.Transactions[0].Patches { + api.stored[patch.AssetID] = append(json.RawMessage(nil), patch.Value...) + versions[patch.AssetID] = 8 + } + if api.applyErr != nil { + return nil, api.applyErr + } + return &canvas.ApplyResult{ + BatchID: opts.Request.BatchID, + Results: []canvas.PatchTransactionResult{{TransactionID: opts.Request.Transactions[0].TransactionID, Status: "ack", AssetVersions: versions}}, + LogID: "apply-log", + }, nil +} + +func (api *fakeCanvasAPI) totalCalls() int { + return api.createCalls + api.resumeCreateCalls + api.allocateCalls + api.getCalls + api.getExistingCalls + api.applyCalls +} + +type testedAsset struct { + ID string + Version int64 + Content json.RawMessage +} + +func queriedAsset(asset testedAsset) json.RawMessage { + content, _ := json.Marshal(string(asset.Content)) + return json.RawMessage(fmt.Sprintf( + `{"PippitAssetID":%q,"Version":%d,"TextInfo":{"Content":%s}}`, + asset.ID, + asset.Version, + content, + )) +} + +func readJournal(t *testing.T, path string) *Journal { + t.Helper() + file, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + journal, err := decodeJournal(file) + if err != nil { + t.Fatal(err) + } + return journal +} + +func decodeRawMap(t *testing.T, raw json.RawMessage) map[string]any { + t.Helper() + var result map[string]any + if err := json.Unmarshal(raw, &result); err != nil { + t.Fatal(err) + } + return result +} + +func asMap(t *testing.T, value any) map[string]any { + t.Helper() + result, ok := value.(map[string]any) + if !ok { + t.Fatalf("value %#v is not an object", value) + } + return result +} + +func asSlice(t *testing.T, value any) []any { + t.Helper() + result, ok := value.([]any) + if !ok { + t.Fatalf("value %#v is not an array", value) + } + return result +} + +func companionContent(t *testing.T, document *Document, assetID string) map[string]any { + t.Helper() + asset := decodeRawMap(t, document.Assets[assetID]) + return asMap(t, asset["content"]) +} diff --git a/internal/canvasplan/journal_security_unix_test.go b/internal/canvasplan/journal_security_unix_test.go new file mode 100644 index 0000000..b5c7ce5 --- /dev/null +++ b/internal/canvasplan/journal_security_unix_test.go @@ -0,0 +1,139 @@ +//go:build !windows + +package canvasplan + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestExecutorRejectsSymlinkJournalWithoutTouchingTarget(t *testing.T) { + directory := t.TempDir() + victim := filepath.Join(directory, "victim.json") + writeVictim(t, victim) + journalPath := filepath.Join(directory, "journal.json") + if err := os.Symlink(victim, journalPath); err != nil { + t.Fatal(err) + } + + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + result, err := (&Executor{api: api}).Execute( + context.Background(), + plan, + resolved, + ExecuteOptions{JournalPath: journalPath}, + ) + if err == nil || result != nil || !strings.Contains(strings.ToLower(err.Error()), "symbolic link") { + t.Fatalf("Execute() result=%#v error=%v, want symlink rejection", result, err) + } + if api.totalCalls() != 0 { + t.Fatalf("remote API was called %d times", api.totalCalls()) + } + assertVictimUnchanged(t, victim) +} + +func TestExecutorRejectsSymlinkLockWithoutTouchingTarget(t *testing.T) { + directory := t.TempDir() + victim := filepath.Join(directory, "lock-victim") + writeVictim(t, victim) + journalPath := filepath.Join(directory, "journal.json") + if err := os.Symlink(victim, journalPath+".lock"); err != nil { + t.Fatal(err) + } + + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + result, err := (&Executor{api: api}).Execute( + context.Background(), + plan, + resolved, + ExecuteOptions{JournalPath: journalPath}, + ) + if err == nil || result != nil || !strings.Contains(strings.ToLower(err.Error()), "symbolic link") { + t.Fatalf("Execute() result=%#v error=%v, want lock symlink rejection", result, err) + } + if api.totalCalls() != 0 { + t.Fatalf("remote API was called %d times", api.totalCalls()) + } + assertVictimUnchanged(t, victim) +} + +func TestExecutorRejectsSymlinkParentDirectory(t *testing.T) { + directory := t.TempDir() + realParent := filepath.Join(directory, "real-parent") + if err := os.Mkdir(realParent, 0o700); err != nil { + t.Fatal(err) + } + linkedParent := filepath.Join(directory, "linked-parent") + if err := os.Symlink(realParent, linkedParent); err != nil { + t.Fatal(err) + } + journalPath := filepath.Join(linkedParent, "journal.json") + + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + result, err := (&Executor{api: api}).Execute( + context.Background(), + plan, + resolved, + ExecuteOptions{JournalPath: journalPath}, + ) + if err == nil || result != nil || !strings.Contains(strings.ToLower(err.Error()), "symbolic link") { + t.Fatalf("Execute() result=%#v error=%v, want parent symlink rejection", result, err) + } + if api.totalCalls() != 0 { + t.Fatalf("remote API was called %d times", api.totalCalls()) + } + for _, unexpected := range []string{filepath.Join(realParent, "journal.json"), filepath.Join(realParent, "journal.json.lock")} { + if _, statErr := os.Lstat(unexpected); !os.IsNotExist(statErr) { + t.Fatalf("unexpected file through symlink %s: %v", unexpected, statErr) + } + } +} + +func TestSaveJournalRejectsSymlinkDestinationWithoutTouchingTarget(t *testing.T) { + directory := t.TempDir() + victim := filepath.Join(directory, "save-victim.json") + writeVictim(t, victim) + journalPath := filepath.Join(directory, "journal.json") + if err := os.Symlink(victim, journalPath); err != nil { + t.Fatal(err) + } + journal := &Journal{Schema: JournalSchema, OperationID: "operation", RequestID: "request", State: StateInitialized} + if err := saveJournal(journalPath, journal); err == nil || !strings.Contains(strings.ToLower(err.Error()), "symbolic link") { + t.Fatalf("saveJournal() error = %v, want symlink rejection", err) + } + assertVictimUnchanged(t, victim) +} + +func writeVictim(t *testing.T, path string) { + t.Helper() + if err := os.WriteFile(path, []byte("do-not-touch\n"), 0o640); err != nil { + t.Fatal(err) + } +} + +func assertVictimUnchanged(t *testing.T, path string) { + t.Helper() + payload, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(payload) != "do-not-touch\n" { + t.Fatalf("victim content changed: %q", payload) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Size() != int64(len(payload)) { + t.Fatalf("victim size changed: %d", info.Size()) + } + if info.Mode().Perm() != 0o640 { + t.Fatalf("victim mode changed: %o", info.Mode().Perm()) + } +} From 1bc6d7376c000092ce2bae348375343f601e1459 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:47:12 +0800 Subject: [PATCH 12/48] feat: isolate LibTV export execution Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_export.go | 381 ++++++++++++++++++++++++++++++++++++ 1 file changed, 381 insertions(+) create mode 100644 cmd/canvas/import_export.go diff --git a/cmd/canvas/import_export.go b/cmd/canvas/import_export.go new file mode 100644 index 0000000..5d87c06 --- /dev/null +++ b/cmd/canvas/import_export.go @@ -0,0 +1,381 @@ +package canvas + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/canvasplan" +) + +const ( + maxLibTVExporterOutputBytes = 4 << 20 + libTVExportResultSchema = "pippit-libtv-export-result/0.1" +) + +type libTVExportResult struct { + BundleDir string `json:"bundle_dir"` + SnapshotPath string `json:"snapshot_path"` + MediaManifestPath string `json:"media_manifest_path"` + PlanPath string `json:"plan_path"` + Schema string `json:"schema"` + PlanSchema string `json:"plan_schema"` + Source canvasplan.Source `json:"source"` + Media []libTVExportMedia `json:"media"` + MediaCount int `json:"media_count"` + NodeCount int `json:"node_count"` + GroupCount int `json:"group_count"` + EdgeCount int `json:"edge_count"` + DegradationCount int `json:"degradation_count"` +} + +type libTVExportMedia struct { + LogicalID string `json:"logical_id"` + MediaType string `json:"media_type"` + LocalPath string `json:"local_path"` +} + +type nodeLibTVExporter struct{} + +func (nodeLibTVExporter) Export( + ctx context.Context, + sourceURL string, + outputDir string, + stderr io.Writer, +) (*libTVExportResult, error) { + root, err := findCLIPackageRoot() + if err != nil { + return nil, err + } + node, err := exec.LookPath("node") + if err != nil { + return nil, fmt.Errorf("find Node.js for LibTV exporter: %w", err) + } + adapterPath := filepath.Join(root, "adapters", "libtv", "cli.mjs") + command := exec.CommandContext(ctx, node, adapterPath, + "export", "--url", sourceURL, "--output-dir", outputDir, + ) + command.Env = sanitizedExporterEnv(os.Environ()) + command.Stdin = os.Stdin + command.Stderr = stderr + var stdout boundedBuffer + stdout.maximum = maxLibTVExporterOutputBytes + command.Stdout = &stdout + if err := command.Run(); err != nil { + if stdout.exceeded { + return nil, fmt.Errorf("LibTV exporter output exceeds %d bytes", maxLibTVExporterOutputBytes) + } + return nil, fmt.Errorf("LibTV exporter failed: %w", err) + } + if stdout.exceeded { + return nil, fmt.Errorf("LibTV exporter output exceeds %d bytes", maxLibTVExporterOutputBytes) + } + var result libTVExportResult + decoder := json.NewDecoder(bytes.NewReader(stdout.Bytes())) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&result); err != nil { + return nil, fmt.Errorf("decode LibTV exporter result: %w", err) + } + if err := ensureImportJSONEOF(decoder); err != nil { + return nil, fmt.Errorf("decode LibTV exporter result: %w", err) + } + return &result, nil +} + +func ensureImportJSONEOF(decoder *json.Decoder) error { + var trailing any + if err := decoder.Decode(&trailing); err == io.EOF { + return nil + } else if err != nil { + return fmt.Errorf("decode trailing JSON: %w", err) + } + return fmt.Errorf("JSON input must contain exactly one value") +} + +type boundedBuffer struct { + bytes.Buffer + maximum int + exceeded bool +} + +func (buffer *boundedBuffer) Write(value []byte) (int, error) { + remaining := buffer.maximum - buffer.Len() + if remaining > 0 { + written := len(value) + if written > remaining { + written = remaining + } + _, _ = buffer.Buffer.Write(value[:written]) + } + if len(value) > remaining { + buffer.exceeded = true + } + return len(value), nil +} + +func sanitizedExporterEnv(environ []string) []string { + result := make([]string, 0, len(environ)) + for _, entry := range environ { + key, value, found := strings.Cut(entry, "=") + if !found { + continue + } + upper := strings.ToUpper(strings.TrimSpace(key)) + if !allowedExporterEnvKey(upper) || !safeExporterEnvValue(upper, value) { + continue + } + result = append(result, entry) + } + return result +} + +func allowedExporterEnvKey(key string) bool { + if strings.HasPrefix(key, "LC_") { + return true + } + switch key { + case "ALL_PROXY", + "APPDATA", + "COLORTERM", + "COMSPEC", + "CURL_CA_BUNDLE", + "DBUS_SESSION_BUS_ADDRESS", + "DISPLAY", + "HOME", + "HTTP_PROXY", + "HTTPS_PROXY", + "LANG", + "LIBTV_CLI_BINARY", + "LIBTV_CLI_PATH", + "LIBTV_CONFIG_DIR", + "LOCALAPPDATA", + "NODE_EXTRA_CA_CERTS", + "NO_PROXY", + "PATH", + "PATHEXT", + "PIPPIT_CLI_LIBTV_CACHE_DIR", + "SHELL", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "SYSTEMROOT", + "TEMP", + "TERM", + "TMP", + "TMPDIR", + "TZ", + "USER", + "USERNAME", + "USERPROFILE", + "WAYLAND_DISPLAY", + "WINDIR", + "XAUTHORITY", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_RUNTIME_DIR": + return true + default: + return false + } +} + +func safeExporterEnvValue(key, value string) bool { + if key == "NO_PROXY" { + return true + } + if key != "HTTP_PROXY" && key != "HTTPS_PROXY" && key != "ALL_PROXY" { + return true + } + if strings.TrimSpace(value) == "" { + return false + } + parsed, err := url.Parse(value) + if err != nil || parsed.User != nil || parsed.Hostname() == "" || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" { + return false + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https", "socks", "socks4", "socks4a", "socks5", "socks5h": + return true + default: + return false + } +} + +func findCLIPackageRoot() (string, error) { + candidates := make([]string, 0, 3) + if configured := strings.TrimSpace(os.Getenv("PIPPIT_CLI_PACKAGE_ROOT")); configured != "" { + candidates = append(candidates, configured) + } + if workingDirectory, err := os.Getwd(); err == nil { + candidates = append(candidates, workingDirectory) + } + if executable, err := os.Executable(); err == nil { + candidates = append(candidates, filepath.Dir(executable)) + } + seen := make(map[string]struct{}) + for _, candidate := range candidates { + absolute, err := filepath.Abs(candidate) + if err != nil { + continue + } + for directory := filepath.Clean(absolute); ; directory = filepath.Dir(directory) { + if _, duplicate := seen[directory]; !duplicate { + seen[directory] = struct{}{} + adapter := filepath.Join(directory, "adapters", "libtv", "cli.mjs") + if info, statErr := os.Stat(adapter); statErr == nil && info.Mode().IsRegular() { + return directory, nil + } + } + parent := filepath.Dir(directory) + if parent == directory { + break + } + } + } + return "", fmt.Errorf("locate packaged LibTV exporter; run through the @pippit-dev/cli entry point or set PIPPIT_CLI_PACKAGE_ROOT") +} + +func newImportBundlePath(userCacheDir func() (string, error)) (string, string, error) { + cacheDir, err := userCacheDir() + if err != nil { + return "", "", fmt.Errorf("resolve canvas import cache directory: %w", err) + } + root := filepath.Join(cacheDir, "pippit-cli", "canvas-import", "exports") + if err := os.MkdirAll(root, 0o700); err != nil { + return "", "", fmt.Errorf("create canvas import export directory: %w", err) + } + if err := os.Chmod(root, 0o700); err != nil { + return "", "", fmt.Errorf("secure canvas import export directory: %w", err) + } + random := make([]byte, 16) + if _, err := rand.Read(random); err != nil { + return "", "", fmt.Errorf("generate canvas import export directory: %w", err) + } + return root, filepath.Join(root, "export-"+hex.EncodeToString(random)), nil +} + +func validateExportLocation(result *libTVExportResult, expectedBundleDir string) error { + if result == nil { + return fmt.Errorf("LibTV exporter returned no result") + } + expected, err := filepath.Abs(expectedBundleDir) + if err != nil { + return fmt.Errorf("resolve expected LibTV bundle path: %w", err) + } + bundle, err := filepath.Abs(strings.TrimSpace(result.BundleDir)) + if err != nil || filepath.Clean(bundle) != filepath.Clean(expected) { + return fmt.Errorf("LibTV exporter returned an unexpected bundle directory") + } + for label, path := range map[string]string{ + "snapshot": result.SnapshotPath, + "media manifest": result.MediaManifestPath, + "CanvasPlan": result.PlanPath, + } { + if err := requireFileWithinBundle(path, bundle); err != nil { + return fmt.Errorf("invalid LibTV %s path: %w", label, err) + } + } + return nil +} + +func requireFileWithinBundle(path, bundle string) error { + if !filepath.IsAbs(strings.TrimSpace(path)) { + return fmt.Errorf("path must be absolute") + } + realBundle, err := filepath.EvalSymlinks(bundle) + if err != nil { + return fmt.Errorf("resolve bundle directory: %w", err) + } + realPath, err := filepath.EvalSymlinks(path) + if err != nil { + return fmt.Errorf("resolve file: %w", err) + } + relative, err := filepath.Rel(realBundle, realPath) + if err != nil || relative == "." || relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return fmt.Errorf("path escapes the export bundle") + } + info, err := os.Stat(realPath) + if err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("path is not a regular file") + } + return nil +} + +func readCanvasPlan(path string) (canvasplan.Plan, error) { + file, err := os.Open(path) + if err != nil { + return canvasplan.Plan{}, fmt.Errorf("open exported CanvasPlan: %w", err) + } + defer file.Close() + plan, err := canvasplan.DecodePlan(file) + if err != nil { + return canvasplan.Plan{}, err + } + return plan, nil +} + +func validateExportPlan(exported libTVExportResult, plan canvasplan.Plan) error { + if exported.Schema != libTVExportResultSchema || exported.PlanSchema != canvasplan.PlanSchema { + return fmt.Errorf("LibTV exporter returned unsupported schema %q", exported.Schema) + } + if exported.Source != plan.Source { + return fmt.Errorf("LibTV exporter source does not match CanvasPlan source") + } + if exported.MediaCount != len(exported.Media) || exported.MediaCount != len(plan.RequiredMedia) { + return fmt.Errorf("LibTV exporter media counts do not match CanvasPlan") + } + if exported.NodeCount != len(plan.Nodes) || exported.GroupCount != len(plan.Groups) || + exported.EdgeCount != len(plan.Edges) || exported.DegradationCount != len(plan.Degradations) { + return fmt.Errorf("LibTV exporter counts do not match CanvasPlan") + } + requirements := make(map[string]canvasplan.MediaRequirement, len(plan.RequiredMedia)) + for _, requirement := range plan.RequiredMedia { + requirements[requirement.LogicalID] = requirement + } + seen := make(map[string]struct{}, len(exported.Media)) + for _, media := range exported.Media { + requirement, ok := requirements[media.LogicalID] + if !ok || requirement.MediaType != media.MediaType { + return fmt.Errorf("LibTV exporter media does not match CanvasPlan") + } + expectedPath, expectedErr := filepath.Abs(filepath.Join( + exported.BundleDir, + filepath.FromSlash(requirement.LocalPath), + )) + actualPath, actualErr := filepath.Abs(strings.TrimSpace(media.LocalPath)) + if expectedErr != nil || actualErr != nil || filepath.Clean(actualPath) != filepath.Clean(expectedPath) { + return fmt.Errorf("LibTV exporter media path does not match CanvasPlan") + } + if _, duplicate := seen[media.LogicalID]; duplicate { + return fmt.Errorf("LibTV exporter returned duplicate media %q", media.LogicalID) + } + seen[media.LogicalID] = struct{}{} + } + return nil +} + +func removeOwnedBundle(bundleDir, bundleRoot string) error { + cleanRoot, err := filepath.Abs(bundleRoot) + if err != nil { + return err + } + cleanBundle, err := filepath.Abs(bundleDir) + if err != nil { + return err + } + relative, err := filepath.Rel(cleanRoot, cleanBundle) + if err != nil || strings.Contains(relative, string(filepath.Separator)) || + !strings.HasPrefix(relative, "export-") { + return fmt.Errorf("refusing to remove an unowned LibTV bundle") + } + return os.RemoveAll(cleanBundle) +} From f8ed45fd4198392eef841e56dd09e56c5c5ffd80 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:47:25 +0800 Subject: [PATCH 13/48] feat: checkpoint canvas media uploads Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_media.go | 555 ++++++++++++++++++++++++ cmd/canvas/import_media_lock.go | 77 ++++ cmd/canvas/import_media_lock_unix.go | 26 ++ cmd/canvas/import_media_lock_windows.go | 57 +++ 4 files changed, 715 insertions(+) create mode 100644 cmd/canvas/import_media.go create mode 100644 cmd/canvas/import_media_lock.go create mode 100644 cmd/canvas/import_media_lock_unix.go create mode 100644 cmd/canvas/import_media_lock_windows.go diff --git a/cmd/canvas/import_media.go b/cmd/canvas/import_media.go new file mode 100644 index 0000000..ce68c12 --- /dev/null +++ b/cmd/canvas/import_media.go @@ -0,0 +1,555 @@ +package canvas + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" + "github.com/Pippit-dev/pippit-cli/internal/canvasplan" + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +const ( + mediaCheckpointSchema = "pippit-canvas-import-media/0.1" + mediaStatusReady = "ready" + mediaStatusProcessing = "processing" + mediaStatusUploadRequested = "upload-requested" + mediaStatusBlocked = "blocked" + mediaStatusBlockedInterruption = "blocked-on-interruption" + maxMediaCheckpointBytes = 8 << 20 +) + +type importMediaPreflighter interface { + PreflightUpload(context.Context) error +} + +type importMediaAPI interface { + Upload(context.Context, string) (*canvascore.UploadResult, error) + Query(context.Context, string) error +} + +type runnerImportMediaAPI struct { + runner *common.Runner +} + +func (api runnerImportMediaAPI) Upload(ctx context.Context, path string) (*canvascore.UploadResult, error) { + return canvascore.Upload(ctx, canvascore.UploadOptions{Path: path}, api.runner) +} + +func (api runnerImportMediaAPI) Query(ctx context.Context, pippitAssetID string) error { + _, err := canvascore.Get(ctx, canvascore.GetOptions{AssetIDs: []string{pippitAssetID}}, api.runner) + return err +} + +// PreflightUpload mirrors the current Access Key authorizer's local guard. It +// runs immediately before the durable upload-requested marker is written, so +// a missing key cannot turn into an ambiguous remote outcome. +func (api runnerImportMediaAPI) PreflightUpload(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + if api.runner == nil || api.runner.Config == nil { + return fmt.Errorf("canvas media uploader is not configured") + } + if strings.TrimSpace(api.runner.Config.AccessKey) == "" { + return fmt.Errorf("XYQ_ACCESS_KEY 缺失; authenticate the Pippit CLI before importing media") + } + return nil +} + +type validatedImportMedia struct { + LogicalID string + MediaType string + LocalPath string + SHA256 string + ByteSize int64 +} + +type mediaResolutionOptions struct { + Plan canvasplan.Plan + Media []validatedImportMedia + Target string + BundleDir string + BundleRoot string + CanvasJournalPath string + CheckpointPath string +} + +type mediaCheckpoint struct { + Schema string `json:"schema"` + Source canvasplan.Source `json:"source"` + Target string `json:"target"` + BundleDirs []string `json:"bundle_dirs,omitempty"` + Entries []mediaCheckpointEntry `json:"entries"` +} + +type mediaCheckpointEntry struct { + LogicalID string `json:"logical_id"` + MediaType string `json:"media_type"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + AssetID string `json:"asset_id,omitempty"` + PippitAssetID string `json:"pippit_asset_id,omitempty"` + LastError string `json:"last_error,omitempty"` +} + +func readAndValidateExportMedia(bundleDir string, plan canvasplan.Plan) ([]validatedImportMedia, error) { + result := make([]validatedImportMedia, 0, len(plan.RequiredMedia)) + for _, requirement := range plan.RequiredMedia { + if requirement.LocalPath == "" || requirement.URL != "" { + return nil, fmt.Errorf("LibTV CanvasPlan media %q must use a local bundle path", requirement.LogicalID) + } + localPath := filepath.Join(bundleDir, filepath.FromSlash(requirement.LocalPath)) + if err := requireFileWithinBundle(localPath, bundleDir); err != nil { + return nil, fmt.Errorf("invalid LibTV media %q: %w", requirement.LogicalID, err) + } + info, err := os.Stat(localPath) + if err != nil { + return nil, fmt.Errorf("inspect LibTV media %q: %w", requirement.LogicalID, err) + } + if requirement.Metadata.ByteSize == nil || info.Size() != *requirement.Metadata.ByteSize { + return nil, fmt.Errorf("LibTV media %q byte size does not match CanvasPlan", requirement.LogicalID) + } + digest, err := fileSHA256(localPath) + if err != nil { + return nil, fmt.Errorf("hash LibTV media %q: %w", requirement.LogicalID, err) + } + if digest != requirement.SHA256 { + return nil, fmt.Errorf("LibTV media %q SHA-256 does not match CanvasPlan", requirement.LogicalID) + } + result = append(result, validatedImportMedia{ + LogicalID: requirement.LogicalID, + MediaType: requirement.MediaType, + LocalPath: localPath, + SHA256: digest, + ByteSize: info.Size(), + }) + } + return result, nil +} + +func fileSHA256(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func resolveImportMedia( + ctx context.Context, + opts mediaResolutionOptions, + api importMediaAPI, + stderr io.Writer, +) (canvasplan.ResolvedMediaSet, error) { + lock, err := acquireImportMediaCheckpointLock(opts.CheckpointPath + ".lock") + if err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + defer func() { + if err := lock.release(); err != nil { + fmt.Fprintf(stderr, "Could not release canvas import media checkpoint lock: %v\n", err) + } + }() + + checkpoint, err := loadMediaCheckpoint(opts) + if err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + if !containsString(checkpoint.BundleDirs, opts.BundleDir) { + checkpoint.BundleDirs = append(checkpoint.BundleDirs, opts.BundleDir) + if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + } + entries, err := validateCheckpointEntries(checkpoint, opts.Media) + if err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + queriedReadyAssetIDs := make(map[string]struct{}) + for _, media := range opts.Media { + if existing := entries[media.LogicalID]; existing != nil { + switch existing.Status { + case mediaStatusBlocked: + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "media upload %q is blocked after an unknown outcome; inspect %s and do not retry the bytes blindly", + media.LogicalID, opts.CheckpointPath, + ) + case mediaStatusUploadRequested: + existing.Status = mediaStatusBlockedInterruption + existing.LastError = "the previous process stopped after persisting upload-requested and before a durable response was checkpointed" + if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, *existing); err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "media upload %q was interrupted with an unknown outcome; checkpointed as blocked-on-interruption in %s and will not be uploaded again automatically", + media.LogicalID, opts.CheckpointPath, + ) + case mediaStatusBlockedInterruption: + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "media upload %q is blocked-on-interruption after an unknown outcome; inspect %s and do not retry the bytes blindly", + media.LogicalID, opts.CheckpointPath, + ) + case mediaStatusProcessing: + fmt.Fprintf(stderr, "Checking previously uploaded media %q...\n", media.LogicalID) + if err := api.Query(ctx, existing.PippitAssetID); err != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "previous media upload %q is not queryable yet; durable IDs remain in %s: %w", + media.LogicalID, opts.CheckpointPath, err, + ) + } + existing.Status = mediaStatusReady + existing.LastError = "" + queriedReadyAssetIDs[existing.PippitAssetID] = struct{}{} + if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, *existing); err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + case mediaStatusReady: + if _, queried := queriedReadyAssetIDs[existing.PippitAssetID]; !queried { + fmt.Fprintf(stderr, "Verifying previously uploaded media %q for the current Pippit account...\n", media.LogicalID) + if err := api.Query(ctx, existing.PippitAssetID); err != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "previously uploaded media %q is unavailable to the current Pippit account; refusing checkpoint reuse: %w", + media.LogicalID, err, + ) + } + queriedReadyAssetIDs[existing.PippitAssetID] = struct{}{} + } + default: + return canvasplan.ResolvedMediaSet{}, fmt.Errorf("media checkpoint %q has invalid status %q", media.LogicalID, existing.Status) + } + continue + } + if duplicate := readyEntryByDigest(entries, media); duplicate != nil { + if _, queried := queriedReadyAssetIDs[duplicate.PippitAssetID]; !queried { + fmt.Fprintf(stderr, "Verifying deduplicated media %q for the current Pippit account...\n", media.LogicalID) + if err := api.Query(ctx, duplicate.PippitAssetID); err != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "deduplicated media %q is unavailable to the current Pippit account; refusing checkpoint reuse: %w", + media.LogicalID, err, + ) + } + queriedReadyAssetIDs[duplicate.PippitAssetID] = struct{}{} + } + entry := mediaCheckpointEntry{ + LogicalID: media.LogicalID, + MediaType: media.MediaType, + SHA256: media.SHA256, + Status: mediaStatusReady, + AssetID: duplicate.AssetID, + PippitAssetID: duplicate.PippitAssetID, + } + entries[media.LogicalID] = &entry + if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + continue + } + if preflighter, ok := api.(importMediaPreflighter); ok { + if err := preflighter.PreflightUpload(ctx); err != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "Pippit authentication failed before media upload %q was requested: %w", + media.LogicalID, err, + ) + } + } + fmt.Fprintf(stderr, "Uploading LibTV media %d/%d...\n", len(entries)+1, len(opts.Media)) + entry := mediaCheckpointEntry{ + LogicalID: media.LogicalID, + MediaType: media.MediaType, + SHA256: media.SHA256, + Status: mediaStatusUploadRequested, + LastError: "upload request is about to be dispatched; interruption requires manual outcome confirmation", + } + if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "persist upload-requested checkpoint for media %q: %w", + media.LogicalID, err, + ) + } + entries[media.LogicalID] = &entry + uploaded, uploadErr := api.Upload(ctx, media.LocalPath) + if uploadErr != nil && strings.Contains(uploadErr.Error(), "XYQ_ACCESS_KEY 缺失") { + if checkpointErr := removeAndSaveMediaEntry(opts.CheckpointPath, checkpoint, media.LogicalID); checkpointErr != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "Pippit authentication failed before media upload %q was sent, but its upload-requested checkpoint could not be cleared; do not retry blindly: %w", + media.LogicalID, checkpointErr, + ) + } + delete(entries, media.LogicalID) + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "Pippit authentication failed before media upload %q was sent: %w", + media.LogicalID, uploadErr, + ) + } + entry.LastError = "" + if uploaded != nil { + entry.AssetID = strings.TrimSpace(uploaded.AssetID) + entry.PippitAssetID = strings.TrimSpace(uploaded.PippitAssetID) + } + if uploadErr != nil || uploaded == nil { + entry.Status = mediaStatusBlocked + entry.LastError = errorText(uploadErr, "upload returned no result") + if checkpointErr := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); checkpointErr != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "media upload %q has an unknown outcome and its blocked checkpoint could not be saved; do not retry the bytes blindly: %w", + media.LogicalID, checkpointErr, + ) + } + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "media upload %q has an unknown outcome; checkpointed as blocked in %s: %s", + media.LogicalID, opts.CheckpointPath, errorText(uploadErr, "upload returned no result"), + ) + } + if entry.AssetID == "" || entry.PippitAssetID == "" { + entry.Status = mediaStatusBlocked + entry.LastError = "upload response omitted durable asset IDs" + if checkpointErr := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); checkpointErr != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "media upload %q omitted durable IDs and its blocked checkpoint could not be saved; do not upload again blindly: %w", + media.LogicalID, checkpointErr, + ) + } + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "media upload %q omitted durable asset IDs; checkpointed as blocked in %s", + media.LogicalID, opts.CheckpointPath, + ) + } + if uploaded.State != canvascore.StateReady { + entry.Status = mediaStatusProcessing + entry.LastError = strings.TrimSpace(uploaded.Warning) + if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "media upload %q is still processing; durable IDs are checkpointed in %s, rerun to query without re-uploading", + media.LogicalID, opts.CheckpointPath, + ) + } + entry.Status = mediaStatusReady + queriedReadyAssetIDs[entry.PippitAssetID] = struct{}{} + entries[media.LogicalID] = &entry + if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + } + resolved := canvasplan.ResolvedMediaSet{Schema: canvasplan.ResolvedMediaSchema} + for _, media := range opts.Media { + entry := entries[media.LogicalID] + resolved.Media = append(resolved.Media, canvasplan.ResolvedMedia{ + LogicalID: entry.LogicalID, + MediaType: entry.MediaType, + AssetID: entry.AssetID, + PippitAssetID: entry.PippitAssetID, + }) + } + resolved, err = canvasplan.NormalizeResolvedMedia(resolved) + if err != nil { + return canvasplan.ResolvedMediaSet{}, err + } + cleanupCheckpointBundles(checkpoint, opts, stderr) + return resolved, nil +} + +func loadMediaCheckpoint(opts mediaResolutionOptions) (*mediaCheckpoint, error) { + if info, lstatErr := os.Lstat(opts.CheckpointPath); lstatErr == nil { + if info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("canvas import media checkpoint must not be a symbolic link") + } + if info.Size() > maxMediaCheckpointBytes { + return nil, fmt.Errorf("canvas import media checkpoint exceeds %d bytes", maxMediaCheckpointBytes) + } + } else if !os.IsNotExist(lstatErr) { + return nil, fmt.Errorf("inspect canvas import media checkpoint: %w", lstatErr) + } + file, err := os.Open(opts.CheckpointPath) + if os.IsNotExist(err) { + if _, journalErr := os.Stat(opts.CanvasJournalPath); journalErr == nil { + return nil, fmt.Errorf("canvas journal exists but its media checkpoint is missing; refusing to upload again: %s", opts.CheckpointPath) + } else if !os.IsNotExist(journalErr) { + return nil, fmt.Errorf("inspect canvas import journal before media upload: %w", journalErr) + } + checkpoint := &mediaCheckpoint{ + Schema: mediaCheckpointSchema, + Source: opts.Plan.Source, + Target: opts.Target, + Entries: []mediaCheckpointEntry{}, + } + if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { + return nil, err + } + return checkpoint, nil + } + if err != nil { + return nil, fmt.Errorf("open canvas import media checkpoint: %w", err) + } + defer file.Close() + decoder := json.NewDecoder(io.LimitReader(file, maxMediaCheckpointBytes+1)) + decoder.DisallowUnknownFields() + var checkpoint mediaCheckpoint + if err := decoder.Decode(&checkpoint); err != nil { + return nil, fmt.Errorf("decode canvas import media checkpoint: %w", err) + } + if err := ensureImportJSONEOF(decoder); err != nil { + return nil, fmt.Errorf("decode canvas import media checkpoint: %w", err) + } + if checkpoint.Schema != mediaCheckpointSchema || checkpoint.Source != opts.Plan.Source || checkpoint.Target != opts.Target { + return nil, fmt.Errorf("canvas import media checkpoint does not match this source and target") + } + if err := os.Chmod(opts.CheckpointPath, 0o600); err != nil { + return nil, fmt.Errorf("secure canvas import media checkpoint: %w", err) + } + return &checkpoint, nil +} + +func validateCheckpointEntries( + checkpoint *mediaCheckpoint, + media []validatedImportMedia, +) (map[string]*mediaCheckpointEntry, error) { + expected := make(map[string]validatedImportMedia, len(media)) + for _, item := range media { + expected[item.LogicalID] = item + } + entries := make(map[string]*mediaCheckpointEntry, len(checkpoint.Entries)) + for index := range checkpoint.Entries { + entry := &checkpoint.Entries[index] + item, ok := expected[entry.LogicalID] + if !ok || item.MediaType != entry.MediaType || item.SHA256 != entry.SHA256 { + return nil, fmt.Errorf("canvas import media changed after checkpoint creation") + } + if _, duplicate := entries[entry.LogicalID]; duplicate { + return nil, fmt.Errorf("canvas import media checkpoint contains duplicate logical ID %q", entry.LogicalID) + } + if (entry.Status == mediaStatusReady || entry.Status == mediaStatusProcessing) && + (strings.TrimSpace(entry.AssetID) == "" || strings.TrimSpace(entry.PippitAssetID) == "") { + return nil, fmt.Errorf("canvas import media checkpoint entry %q has no durable IDs", entry.LogicalID) + } + entries[entry.LogicalID] = entry + } + return entries, nil +} + +func readyEntryByDigest(entries map[string]*mediaCheckpointEntry, media validatedImportMedia) *mediaCheckpointEntry { + for _, entry := range entries { + if entry.Status == mediaStatusReady && entry.MediaType == media.MediaType && entry.SHA256 == media.SHA256 { + return entry + } + } + return nil +} + +func replaceAndSaveMediaEntry(path string, checkpoint *mediaCheckpoint, entry mediaCheckpointEntry) error { + replaced := false + for index := range checkpoint.Entries { + if checkpoint.Entries[index].LogicalID == entry.LogicalID { + checkpoint.Entries[index] = entry + replaced = true + break + } + } + if !replaced { + checkpoint.Entries = append(checkpoint.Entries, entry) + } + sort.Slice(checkpoint.Entries, func(i, j int) bool { + return checkpoint.Entries[i].LogicalID < checkpoint.Entries[j].LogicalID + }) + return saveMediaCheckpoint(path, checkpoint) +} + +func removeAndSaveMediaEntry(path string, checkpoint *mediaCheckpoint, logicalID string) error { + entries := make([]mediaCheckpointEntry, 0, len(checkpoint.Entries)) + for _, entry := range checkpoint.Entries { + if entry.LogicalID != logicalID { + entries = append(entries, entry) + } + } + checkpoint.Entries = entries + return saveMediaCheckpoint(path, checkpoint) +} + +func saveMediaCheckpoint(path string, checkpoint *mediaCheckpoint) error { + payload, err := json.MarshalIndent(checkpoint, "", " ") + if err != nil { + return fmt.Errorf("encode canvas import media checkpoint: %w", err) + } + payload = append(payload, '\n') + directory := filepath.Dir(path) + if err := os.MkdirAll(directory, 0o700); err != nil { + return fmt.Errorf("create canvas import media checkpoint directory: %w", err) + } + temporary, err := os.CreateTemp(directory, ".canvas-import-media-*") + if err != nil { + return fmt.Errorf("create temporary canvas import media checkpoint: %w", err) + } + temporaryPath := temporary.Name() + defer func() { + _ = temporary.Close() + _ = os.Remove(temporaryPath) + }() + if err := temporary.Chmod(0o600); err != nil { + return fmt.Errorf("secure temporary canvas import media checkpoint: %w", err) + } + if _, err := temporary.Write(payload); err != nil { + return fmt.Errorf("write canvas import media checkpoint: %w", err) + } + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync canvas import media checkpoint: %w", err) + } + if err := temporary.Close(); err != nil { + return fmt.Errorf("close canvas import media checkpoint: %w", err) + } + if err := os.Rename(temporaryPath, path); err != nil { + return fmt.Errorf("replace canvas import media checkpoint: %w", err) + } + if err := os.Chmod(path, 0o600); err != nil { + return fmt.Errorf("secure canvas import media checkpoint: %w", err) + } + return nil +} + +func cleanupCheckpointBundles( + checkpoint *mediaCheckpoint, + opts mediaResolutionOptions, + stderr io.Writer, +) { + remaining := make([]string, 0, len(checkpoint.BundleDirs)) + for _, bundleDir := range checkpoint.BundleDirs { + if err := removeOwnedBundle(bundleDir, opts.BundleRoot); err != nil { + fmt.Fprintf(stderr, "Could not remove local LibTV export bundle %s: %v\n", bundleDir, err) + remaining = append(remaining, bundleDir) + } + } + checkpoint.BundleDirs = remaining + if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { + fmt.Fprintf(stderr, "Could not update media checkpoint cleanup state: %v\n", err) + } +} + +func containsString(values []string, expected string) bool { + for _, value := range values { + if value == expected { + return true + } + } + return false +} + +func errorText(err error, fallback string) string { + if err == nil { + return fallback + } + return err.Error() +} diff --git a/cmd/canvas/import_media_lock.go b/cmd/canvas/import_media_lock.go new file mode 100644 index 0000000..d4fb4da --- /dev/null +++ b/cmd/canvas/import_media_lock.go @@ -0,0 +1,77 @@ +package canvas + +import ( + "fmt" + "os" + "path/filepath" +) + +type importMediaCheckpointLock struct { + file *os.File + unlock func() error +} + +func acquireImportMediaCheckpointLock(path string) (*importMediaCheckpointLock, error) { + directory := filepath.Dir(path) + if err := os.MkdirAll(directory, 0o700); err != nil { + return nil, fmt.Errorf("create canvas import media lock directory: %w", err) + } + if info, err := os.Lstat(path); err == nil { + if info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("canvas import media checkpoint lock must not be a symbolic link") + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("canvas import media checkpoint lock must be a regular file") + } + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("inspect canvas import media checkpoint lock: %w", err) + } + file, err := openImportMediaLockFile(path) + if err != nil { + return nil, fmt.Errorf("open canvas import media checkpoint lock: %w", err) + } + fileInfo, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, fmt.Errorf("inspect opened canvas import media checkpoint lock: %w", err) + } + pathInfo, err := os.Lstat(path) + if err != nil { + _ = file.Close() + return nil, fmt.Errorf("reinspect canvas import media checkpoint lock: %w", err) + } + if pathInfo.Mode()&os.ModeSymlink != 0 { + _ = file.Close() + return nil, fmt.Errorf("canvas import media checkpoint lock must not be a symbolic link") + } + if !fileInfo.Mode().IsRegular() || !pathInfo.Mode().IsRegular() || !os.SameFile(fileInfo, pathInfo) { + _ = file.Close() + return nil, fmt.Errorf("canvas import media checkpoint lock changed while it was being opened") + } + if err := file.Chmod(0o600); err != nil { + _ = file.Close() + return nil, fmt.Errorf("secure canvas import media checkpoint lock: %w", err) + } + unlock, err := lockImportMediaFile(file) + if err != nil { + _ = file.Close() + return nil, fmt.Errorf("canvas import media checkpoint is locked by another process: %w", err) + } + return &importMediaCheckpointLock{file: file, unlock: unlock}, nil +} + +func (lock *importMediaCheckpointLock) release() error { + if lock == nil { + return nil + } + var unlockErr error + if lock.unlock != nil { + unlockErr = lock.unlock() + } + if lock.file != nil { + if closeErr := lock.file.Close(); unlockErr == nil { + unlockErr = closeErr + } + } + return unlockErr +} diff --git a/cmd/canvas/import_media_lock_unix.go b/cmd/canvas/import_media_lock_unix.go new file mode 100644 index 0000000..e1d9999 --- /dev/null +++ b/cmd/canvas/import_media_lock_unix.go @@ -0,0 +1,26 @@ +//go:build !windows + +package canvas + +import ( + "fmt" + "os" + "syscall" +) + +func openImportMediaLockFile(path string) (*os.File, error) { + fd, err := syscall.Open(path, syscall.O_CREAT|syscall.O_RDWR|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return nil, fmt.Errorf("open lock without following symbolic links: %w", err) + } + return os.NewFile(uintptr(fd), path), nil +} + +func lockImportMediaFile(file *os.File) (func() error, error) { + if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + return nil, err + } + return func() error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_UN) + }, nil +} diff --git a/cmd/canvas/import_media_lock_windows.go b/cmd/canvas/import_media_lock_windows.go new file mode 100644 index 0000000..a643028 --- /dev/null +++ b/cmd/canvas/import_media_lock_windows.go @@ -0,0 +1,57 @@ +//go:build windows + +package canvas + +import ( + "fmt" + "os" + + "golang.org/x/sys/windows" +) + +func openImportMediaLockFile(path string) (*os.File, error) { + pathPointer, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + handle, err := windows.CreateFile( + pathPointer, + windows.GENERIC_READ|windows.GENERIC_WRITE, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + windows.OPEN_ALWAYS, + windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return nil, err + } + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + _ = windows.CloseHandle(handle) + return nil, fmt.Errorf("inspect lock file attributes: %w", err) + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + _ = windows.CloseHandle(handle) + return nil, fmt.Errorf("canvas import media checkpoint lock must not be a symbolic link or reparse point") + } + return os.NewFile(uintptr(handle), path), nil +} + +func lockImportMediaFile(file *os.File) (func() error, error) { + overlapped := &windows.Overlapped{} + handle := windows.Handle(file.Fd()) + if err := windows.LockFileEx( + handle, + windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, + 0, + 1, + 0, + overlapped, + ); err != nil { + return nil, err + } + return func() error { + return windows.UnlockFileEx(handle, 0, 1, 0, overlapped) + }, nil +} From 24258bc33d0be7ad5a5907c14f33ae47ce50fe5e Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:47:35 +0800 Subject: [PATCH 14/48] feat: add one-command canvas import Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/canvas.go | 1 + cmd/canvas/canvas_test.go | 8 +- cmd/canvas/import.go | 352 ++++++++++++++++++++++++++++++++++++++ scripts/run.js | 10 +- 4 files changed, 366 insertions(+), 5 deletions(-) create mode 100644 cmd/canvas/import.go diff --git a/cmd/canvas/canvas.go b/cmd/canvas/canvas.go index 5a86064..cf8b3c2 100644 --- a/cmd/canvas/canvas.go +++ b/cmd/canvas/canvas.go @@ -33,6 +33,7 @@ func NewCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command cmd.AddCommand(newCreateCommand(stdout, stderr, runner)) cmd.AddCommand(newGetCommand(stdout, stderr, runner)) cmd.AddCommand(newApplyCommand(stdout, stderr, runner)) + cmd.AddCommand(newImportCommand(stdout, stderr, newImportDependencies(runner))) cmd.AddCommand(newUploadCommand(stdout, stderr, runner)) return cmd } diff --git a/cmd/canvas/canvas_test.go b/cmd/canvas/canvas_test.go index cd9636a..25a6cb6 100644 --- a/cmd/canvas/canvas_test.go +++ b/cmd/canvas/canvas_test.go @@ -33,16 +33,16 @@ func (f *commandFakeClient) SendMultipartRequest(context.Context, string, map[st return nil } -func TestCommandExposesOnlyProviderNeutralPublicVerbs(t *testing.T) { +func TestCommandExposesCanvasPrimitivesAndImportOrchestration(t *testing.T) { cmd := NewCommand(&bytes.Buffer{}, &bytes.Buffer{}, &common.Runner{Client: &commandFakeClient{}}) got := make([]string, 0, len(cmd.Commands())) for _, child := range cmd.Commands() { got = append(got, child.Name()) } - if strings.Join(got, ",") != "apply,create,get,upload" { - t.Fatalf("commands = %v, want apply/create/get/upload", got) + if strings.Join(got, ",") != "apply,create,get,import,upload" { + t.Fatalf("commands = %v, want apply/create/get/import/upload", got) } - for _, forbidden := range []string{"import", "bind", "team", "libtv", "allocate"} { + for _, forbidden := range []string{"bind", "team", "libtv", "allocate"} { if strings.Contains(strings.ToLower(cmd.CommandPath()+" "+cmd.Short+" "+strings.Join(got, " ")), forbidden) { t.Fatalf("public command surface contains forbidden verb %q", forbidden) } diff --git a/cmd/canvas/import.go b/cmd/canvas/import.go new file mode 100644 index 0000000..54d7b0e --- /dev/null +++ b/cmd/canvas/import.go @@ -0,0 +1,352 @@ +package canvas + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/canvasplan" + "github.com/Pippit-dev/pippit-cli/internal/common" + "github.com/spf13/cobra" +) + +type importOptions struct { + Provider string + SourceURL string + Open bool + AcceptDegradations bool + JournalPath string +} + +var ( + libTVProjectIDPattern = regexp.MustCompile(`^(?:[0-9a-fA-F]{32}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$`) + libTVSpaceIDPattern = regexp.MustCompile(`^[0-9]+$`) +) + +type importExporter interface { + Export(context.Context, string, string, io.Writer) (*libTVExportResult, error) +} + +type importExecutor interface { + Execute(context.Context, canvasplan.Plan, canvasplan.ResolvedMediaSet, canvasplan.ExecuteOptions) (*canvasplan.ExecutionResult, error) +} + +type importDependencies struct { + exporter importExporter + media importMediaAPI + executor importExecutor + openURL func(context.Context, string) error + userCacheDir func() (string, error) + userConfigDir func() (string, error) + target func() string + authScope func() string +} + +type runnerImportExecutor struct { + executor *canvasplan.Executor +} + +func (executor runnerImportExecutor) Execute( + ctx context.Context, + plan canvasplan.Plan, + resolved canvasplan.ResolvedMediaSet, + opts canvasplan.ExecuteOptions, +) (*canvasplan.ExecutionResult, error) { + return executor.executor.Execute(ctx, plan, resolved, opts) +} + +func newImportDependencies(runner *common.Runner) importDependencies { + return importDependencies{ + exporter: nodeLibTVExporter{}, + media: runnerImportMediaAPI{runner: runner}, + executor: runnerImportExecutor{executor: canvasplan.NewExecutor(runner)}, + openURL: openBrowserURL, + userCacheDir: os.UserCacheDir, + userConfigDir: os.UserConfigDir, + target: func() string { return canvasImportTarget(runner) }, + authScope: func() string { return canvasImportAuthScope(runner) }, + } +} + +func newImportCommand( + stdout, stderr io.Writer, + dependencies importDependencies, +) *cobra.Command { + var opts importOptions + cmd := &cobra.Command{ + Use: "import", + Short: "Import an external project into a personal novel Canvas", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + result, err := runCanvasImport(cmd.Context(), opts, dependencies, stderr) + if result != nil { + if writeErr := common.WriteJSON(stdout, result); writeErr != nil { + return writeErr + } + } + if err != nil { + logCanvasError("canvas import", err, map[string]string{ + "provider": strings.TrimSpace(opts.Provider), + "journal": filepath.Base(strings.TrimSpace(opts.JournalPath)), + }) + return err + } + return nil + }, + } + cmd.SetOut(stdout) + cmd.SetErr(stderr) + flags := cmd.Flags() + flags.StringVar(&opts.Provider, "from", "", "source provider (currently: libtv)") + flags.StringVar(&opts.SourceURL, "url", "", "source project URL") + flags.BoolVar(&opts.Open, "open", false, "open the verified personal novel Canvas") + flags.BoolVar(&opts.AcceptDegradations, "accept-degradations", false, "accept explicitly reported source conversion degradations") + flags.StringVar(&opts.JournalPath, "journal", "", "durable resume journal path (generated when omitted)") + return cmd +} + +func runCanvasImport( + ctx context.Context, + opts importOptions, + dependencies importDependencies, + stderr io.Writer, +) (*canvasplan.ExecutionResult, error) { + if strings.ToLower(strings.TrimSpace(opts.Provider)) != "libtv" { + return nil, fmt.Errorf("canvas import --from must be libtv") + } + sourceURL, err := normalizeLibTVURL(opts.SourceURL) + if err != nil { + return nil, err + } + bundleRoot, outputDir, err := newImportBundlePath(dependencies.userCacheDir) + if err != nil { + return nil, err + } + fmt.Fprintln(stderr, "Exporting the LibTV canvas and its media...") + exported, err := dependencies.exporter.Export(ctx, sourceURL, outputDir, stderr) + if err != nil { + return nil, fmt.Errorf("export LibTV canvas: %w", err) + } + if err := validateExportLocation(exported, outputDir); err != nil { + _ = removeOwnedBundle(outputDir, bundleRoot) + return nil, err + } + plan, err := readCanvasPlan(exported.PlanPath) + if err != nil { + _ = removeOwnedBundle(outputDir, bundleRoot) + return nil, err + } + if err := validateExportPlan(*exported, plan); err != nil { + _ = removeOwnedBundle(outputDir, bundleRoot) + return nil, err + } + if len(plan.Degradations) > 0 && !opts.AcceptDegradations { + return nil, fmt.Errorf( + "LibTV export reports %d explicit degradation(s); inspect %s (plan: %s), then rerun with --accept-degradations", + len(plan.Degradations), outputDir, exported.PlanPath, + ) + } + media, err := readAndValidateExportMedia(exported.BundleDir, plan) + if err != nil { + _ = removeOwnedBundle(outputDir, bundleRoot) + return nil, err + } + target := dependencies.target() + authScope := "" + if dependencies.authScope != nil { + authScope = dependencies.authScope() + } + journalPath, err := resolveImportJournalPath( + opts.JournalPath, + plan.Source, + target, + authScope, + dependencies.userConfigDir, + ) + if err != nil { + _ = removeOwnedBundle(outputDir, bundleRoot) + return nil, err + } + checkpointPath := journalPath + ".media.json" + resolved, err := resolveImportMedia(ctx, mediaResolutionOptions{ + Plan: plan, + Media: media, + Target: target, + BundleDir: outputDir, + BundleRoot: bundleRoot, + CanvasJournalPath: journalPath, + CheckpointPath: checkpointPath, + }, dependencies.media, stderr) + if err != nil { + return nil, err + } + fmt.Fprintln(stderr, "Creating or resuming the personal novel Canvas transaction...") + result, executeErr := dependencies.executor.Execute(ctx, plan, resolved, canvasplan.ExecuteOptions{ + JournalPath: journalPath, + }) + if executeErr != nil { + return result, fmt.Errorf("execute CanvasPlan: %w", executeErr) + } + if !verifiedExecution(result) { + return result, fmt.Errorf("CanvasPlan execution completed without query-back verification") + } + if opts.Open { + if err := validateTrustedCanvasURL(result); err != nil { + return result, err + } + if err := dependencies.openURL(ctx, result.WebURL); err != nil { + fmt.Fprintf(stderr, "Canvas verified, but could not open the browser: %v\n", err) + } + } + fmt.Fprintln(stderr, "Canvas import verified.") + return result, nil +} + +func normalizeLibTVURL(value string) (string, error) { + raw := strings.TrimSpace(value) + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme != "https" || parsed.User != nil { + return "", fmt.Errorf("canvas import --url must be an HTTPS LibTV canvas URL") + } + host := strings.ToLower(parsed.Hostname()) + if (host != "www.liblib.tv" && host != "liblib.tv") || (parsed.Port() != "" && parsed.Port() != "443") { + return "", fmt.Errorf("canvas import --url host must be www.liblib.tv") + } + if strings.TrimRight(parsed.EscapedPath(), "/") != "/canvas" { + return "", fmt.Errorf("canvas import --url must identify a LibTV /canvas project with projectId") + } + if parsed.Fragment != "" { + return "", fmt.Errorf("canvas import --url must not contain a fragment") + } + query := parsed.Query() + projectIDs := query["projectId"] + if len(projectIDs) != 1 || !libTVProjectIDPattern.MatchString(strings.TrimSpace(projectIDs[0])) { + return "", fmt.Errorf("canvas import --url projectId must be a LibTV project UUID") + } + canonical := &url.URL{Scheme: "https", Host: "www.liblib.tv", Path: "/canvas"} + canonicalQuery := url.Values{} + canonicalQuery.Set("projectId", strings.ToLower(strings.TrimSpace(projectIDs[0]))) + if spaceIDs, exists := query["spaceId"]; exists { + if len(spaceIDs) != 1 || !libTVSpaceIDPattern.MatchString(strings.TrimSpace(spaceIDs[0])) { + return "", fmt.Errorf("canvas import --url spaceId must be numeric") + } + canonicalQuery.Set("spaceId", strings.TrimSpace(spaceIDs[0])) + } + canonical.RawQuery = canonicalQuery.Encode() + return canonical.String(), nil +} + +func resolveImportJournalPath( + explicit string, + source canvasplan.Source, + target string, + authScope string, + userConfigDir func() (string, error), +) (string, error) { + if value := strings.TrimSpace(explicit); value != "" { + absolute, err := filepath.Abs(value) + if err != nil { + return "", fmt.Errorf("resolve canvas import journal: %w", err) + } + return filepath.Clean(absolute), nil + } + configDir, err := userConfigDir() + if err != nil { + return "", fmt.Errorf("resolve canvas import config directory: %w", err) + } + directory := filepath.Join(configDir, "pippit-cli", "canvas-import") + if err := os.MkdirAll(directory, 0o700); err != nil { + return "", fmt.Errorf("create canvas import journal directory: %w", err) + } + if err := os.Chmod(directory, 0o700); err != nil { + return "", fmt.Errorf("secure canvas import journal directory: %w", err) + } + hash := sha256.Sum256([]byte(strings.Join([]string{ + target, + authScope, + source.Provider, + source.ProjectID, + source.Fingerprint, + }, "\n"))) + return filepath.Join(directory, hex.EncodeToString(hash[:])+".journal.json"), nil +} + +func canvasImportAuthScope(runner *common.Runner) string { + accessKey := "" + if runner != nil && runner.Config != nil { + accessKey = strings.TrimSpace(runner.Config.AccessKey) + } + hash := sha256.Sum256([]byte(accessKey)) + return hex.EncodeToString(hash[:]) +} + +func canvasImportTarget(runner *common.Runner) string { + if runner == nil || runner.Config == nil { + return "unknown" + } + lane := strings.TrimSpace(runner.Config.PPEEnv) + if lane == "" { + lane = "prod" + } + return strings.TrimRight(strings.TrimSpace(runner.Config.BaseURL), "/") + "|" + lane +} + +func verifiedExecution(result *canvasplan.ExecutionResult) bool { + return result != nil && result.State == canvasplan.StateVerified && + result.Verification != nil && result.Verification.Verified +} + +func validateTrustedCanvasURL(result *canvasplan.ExecutionResult) error { + if !verifiedExecution(result) { + return fmt.Errorf("refusing to open an unverified Canvas result") + } + parsed, err := url.Parse(strings.TrimSpace(result.WebURL)) + if err != nil || parsed.Scheme != "https" || parsed.User != nil || + strings.ToLower(parsed.Hostname()) != "xyq.jianying.com" || + (parsed.Port() != "" && parsed.Port() != "443") || parsed.Fragment != "" { + return fmt.Errorf("refusing to open untrusted Canvas URL") + } + if strings.TrimRight(parsed.EscapedPath(), "/") != "/novel/detail/canvas" { + return fmt.Errorf("refusing to open a non-novel Canvas URL") + } + query := parsed.Query() + if len(query["projectId"]) != 1 || query.Get("projectId") != result.ProjectID { + return fmt.Errorf("refusing to open a Canvas URL whose project ID does not match the verified result") + } + if canvasIDs := query["canvasId"]; len(canvasIDs) > 1 || + (len(canvasIDs) == 1 && canvasIDs[0] != result.RootCanvasID) { + return fmt.Errorf("refusing to open a Canvas URL whose canvas ID does not match the verified result") + } + for _, key := range []string{"overviewPippitAssetId", "overview_pippit_asset_id"} { + if overviewIDs := query[key]; len(overviewIDs) > 1 || + (len(overviewIDs) == 1 && overviewIDs[0] != result.OverviewPippitAssetID) { + return fmt.Errorf("refusing to open a Canvas URL whose overview ID does not match the verified result") + } + } + return nil +} + +func openBrowserURL(ctx context.Context, value string) error { + var command *exec.Cmd + switch runtime.GOOS { + case "darwin": + command = exec.CommandContext(ctx, "open", value) + case "windows": + command = exec.CommandContext(ctx, "rundll32", "url.dll,FileProtocolHandler", value) + default: + command = exec.CommandContext(ctx, "xdg-open", value) + } + if err := command.Run(); err != nil { + return fmt.Errorf("open Canvas URL: %w", err) + } + return nil +} diff --git a/scripts/run.js b/scripts/run.js index d569437..a36a983 100755 --- a/scripts/run.js +++ b/scripts/run.js @@ -73,7 +73,15 @@ if (args[0] === "install") { } try { - execFileSync(bin, args, { stdio: "inherit" }); + execFileSync(bin, args, { + stdio: "inherit", + env: { + ...process.env, + // Lets the native command find package-owned adapters after npm has + // installed the binary into a user cache outside this directory. + PIPPIT_CLI_PACKAGE_ROOT: path.join(__dirname, ".."), + }, + }); } catch (e) { process.exit(e.status || 1); } From ca6c4941e46c9e47104d5d0fdf9a96592ce9d2e1 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:47:50 +0800 Subject: [PATCH 15/48] test: cover one-command canvas import Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_test.go | 607 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 607 insertions(+) create mode 100644 cmd/canvas/import_test.go diff --git a/cmd/canvas/import_test.go b/cmd/canvas/import_test.go new file mode 100644 index 0000000..c878ca6 --- /dev/null +++ b/cmd/canvas/import_test.go @@ -0,0 +1,607 @@ +package canvas + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" + "github.com/Pippit-dev/pippit-cli/internal/canvasplan" + "github.com/Pippit-dev/pippit-cli/internal/common" + "github.com/Pippit-dev/pippit-cli/internal/config" +) + +type fakeImportExporter struct { + plan canvasplan.Plan + mediaBytes map[string][]byte + urls []string + bundles []string +} + +func (exporter *fakeImportExporter) Export( + _ context.Context, + sourceURL, outputDir string, + _ io.Writer, +) (*libTVExportResult, error) { + exporter.urls = append(exporter.urls, sourceURL) + exporter.bundles = append(exporter.bundles, outputDir) + if err := os.MkdirAll(filepath.Join(outputDir, "media"), 0o700); err != nil { + return nil, err + } + for relative, payload := range exporter.mediaBytes { + path := filepath.Join(outputDir, filepath.FromSlash(relative)) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, err + } + if err := os.WriteFile(path, payload, 0o600); err != nil { + return nil, err + } + } + planPath := filepath.Join(outputDir, "plan.json") + if err := writeTestJSON(planPath, exporter.plan); err != nil { + return nil, err + } + snapshotPath := filepath.Join(outputDir, "snapshot.json") + if err := writeTestJSON(snapshotPath, map[string]string{"schema": "test"}); err != nil { + return nil, err + } + manifestPath := filepath.Join(outputDir, "media-manifest.json") + if err := writeTestJSON(manifestPath, map[string]any{"media": []any{}}); err != nil { + return nil, err + } + media := make([]libTVExportMedia, 0, len(exporter.plan.RequiredMedia)) + for _, requirement := range exporter.plan.RequiredMedia { + media = append(media, libTVExportMedia{ + LogicalID: requirement.LogicalID, + MediaType: requirement.MediaType, + LocalPath: filepath.Join(outputDir, filepath.FromSlash(requirement.LocalPath)), + }) + } + return &libTVExportResult{ + BundleDir: outputDir, + SnapshotPath: snapshotPath, + MediaManifestPath: manifestPath, + PlanPath: planPath, + Schema: libTVExportResultSchema, + PlanSchema: canvasplan.PlanSchema, + Source: exporter.plan.Source, + Media: media, + MediaCount: len(exporter.plan.RequiredMedia), + NodeCount: len(exporter.plan.Nodes), + GroupCount: len(exporter.plan.Groups), + EdgeCount: len(exporter.plan.Edges), + DegradationCount: len(exporter.plan.Degradations), + }, nil +} + +type fakeImportMediaAPI struct { + uploads int + queries int + uploadState string + uploadErr error + queryErr error +} + +func (api *fakeImportMediaAPI) Upload(_ context.Context, _ string) (*canvascore.UploadResult, error) { + api.uploads++ + if api.uploadErr != nil { + return nil, api.uploadErr + } + state := api.uploadState + if state == "" { + state = canvascore.StateReady + } + return &canvascore.UploadResult{ + State: state, AssetID: "asset-1", PippitAssetID: "pippit-1", + }, nil +} + +func (api *fakeImportMediaAPI) Query(context.Context, string) error { + api.queries++ + return api.queryErr +} + +type panickingImportMediaAPI struct { + uploads int +} + +func (api *panickingImportMediaAPI) Upload(context.Context, string) (*canvascore.UploadResult, error) { + api.uploads++ + panic("simulated process exit during upload") +} + +func (*panickingImportMediaAPI) Query(context.Context, string) error { + return nil +} + +type missingAKPreflightMediaAPI struct { + uploads int +} + +func (*missingAKPreflightMediaAPI) PreflightUpload(context.Context) error { + return errors.New("XYQ_ACCESS_KEY 缺失") +} + +func (api *missingAKPreflightMediaAPI) Upload(context.Context, string) (*canvascore.UploadResult, error) { + api.uploads++ + return nil, errors.New("must not be called") +} + +func (*missingAKPreflightMediaAPI) Query(context.Context, string) error { + return nil +} + +type blockingImportMediaAPI struct { + started chan struct{} + release chan struct{} + uploads int +} + +func (api *blockingImportMediaAPI) Upload(context.Context, string) (*canvascore.UploadResult, error) { + api.uploads++ + close(api.started) + <-api.release + return &canvascore.UploadResult{ + State: canvascore.StateReady, AssetID: "asset-blocking", PippitAssetID: "pippit-blocking", + }, nil +} + +func (*blockingImportMediaAPI) Query(context.Context, string) error { + return nil +} + +type fakeImportExecutor struct { + calls int + resolved canvasplan.ResolvedMediaSet + opts canvasplan.ExecuteOptions + result *canvasplan.ExecutionResult +} + +func (executor *fakeImportExecutor) Execute( + _ context.Context, + _ canvasplan.Plan, + resolved canvasplan.ResolvedMediaSet, + opts canvasplan.ExecuteOptions, +) (*canvasplan.ExecutionResult, error) { + executor.calls++ + executor.resolved = resolved + executor.opts = opts + return executor.result, nil +} + +func TestImportCommandExportsUploadsDeduplicatesVerifiesAndOpens(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &fakeImportMediaAPI{} + executor := &fakeImportExecutor{result: verifiedImportResult()} + opened := "" + deps := testImportDependencies(temp, exporter, media, executor) + deps.openURL = func(_ context.Context, value string) error { opened = value; return nil } + var stdout, stderr bytes.Buffer + cmd := newImportCommand(&stdout, &stderr, deps) + cmd.SilenceUsage = true + cmd.SetArgs([]string{ + "--from", "libtv", + "--url", "https://liblib.tv/canvas?token=secret&spaceId=3872811&projectId=037A5C49E1B344E5ADBC899AD93FDCA9", + "--open", + }) + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v, stderr = %s", err, stderr.String()) + } + if got := exporter.urls[0]; got != "https://www.liblib.tv/canvas?projectId=037a5c49e1b344e5adbc899ad93fdca9&spaceId=3872811" { + t.Fatalf("export URL = %q, want canonical URL without token", got) + } + if media.uploads != 1 { + t.Fatalf("uploads = %d, want one upload for duplicate bytes", media.uploads) + } + if len(executor.resolved.Media) != 2 || executor.resolved.Media[0].PippitAssetID != executor.resolved.Media[1].PippitAssetID { + t.Fatalf("resolved media = %#v, want two logical IDs sharing one uploaded asset", executor.resolved.Media) + } + if opened != executor.result.WebURL { + t.Fatalf("opened = %q, want verified web URL", opened) + } + if strings.Count(stdout.String(), "\n") != 1 || !json.Valid(bytes.TrimSpace(stdout.Bytes())) { + t.Fatalf("stdout = %q, want one JSON line", stdout.String()) + } + if _, err := os.Stat(exporter.bundles[0]); !os.IsNotExist(err) { + t.Fatalf("successful export bundle still exists: %v", err) + } + checkpointInfo, err := os.Stat(executor.opts.JournalPath + ".media.json") + if err != nil || checkpointInfo.Mode().Perm() != 0o600 { + t.Fatalf("media checkpoint info = (%v, %v), want mode 0600", checkpointInfo, err) + } +} + +func TestImportCommandRequiresExplicitDegradationAcceptanceAndKeepsBundle(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, true) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &fakeImportMediaAPI{} + executor := &fakeImportExecutor{result: verifiedImportResult()} + deps := testImportDependencies(temp, exporter, media, executor) + var stdout, stderr bytes.Buffer + cmd := newImportCommand(&stdout, &stderr, deps) + cmd.SilenceUsage = true + cmd.SetArgs([]string{"--from", "libtv", "--url", testLibTVURL}) + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "--accept-degradations") || !strings.Contains(err.Error(), exporter.bundles[0]) { + t.Fatalf("Execute() error = %v, want inspectable degradation gate", err) + } + if media.uploads != 0 || executor.calls != 0 || stdout.Len() != 0 { + t.Fatalf("side effects/stdout = (%d, %d, %q), want none", media.uploads, executor.calls, stdout.String()) + } + if _, err := os.Stat(exporter.bundles[0]); err != nil { + t.Fatalf("degraded export bundle was removed: %v", err) + } +} + +func TestImportCommandCheckpointsProcessingUploadAndDoesNotUploadAgain(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testSingleMediaPlan(t) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &fakeImportMediaAPI{uploadState: canvascore.StateProcessing} + executor := &fakeImportExecutor{result: verifiedImportResult()} + deps := testImportDependencies(temp, exporter, media, executor) + var stdout, stderr bytes.Buffer + first := newImportCommand(&stdout, &stderr, deps) + first.SilenceUsage = true + first.SetArgs([]string{"--from", "libtv", "--url", testLibTVURL}) + if err := first.Execute(); err == nil || !strings.Contains(err.Error(), "still processing") { + t.Fatalf("first Execute() error = %v, want processing checkpoint", err) + } + media.uploadState = canvascore.StateReady + stdout.Reset() + stderr.Reset() + second := newImportCommand(&stdout, &stderr, deps) + second.SetArgs([]string{"--from", "libtv", "--url", testLibTVURL}) + if err := second.Execute(); err != nil { + t.Fatalf("second Execute() error = %v, stderr = %s", err, stderr.String()) + } + if media.uploads != 1 || media.queries != 1 || executor.calls != 1 { + t.Fatalf("upload/query/execute = %d/%d/%d, want 1/1/1", media.uploads, media.queries, executor.calls) + } +} + +func TestImportCommandBlocksUnknownUploadOutcome(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testSingleMediaPlan(t) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &fakeImportMediaAPI{uploadErr: errors.New("connection reset")} + executor := &fakeImportExecutor{result: verifiedImportResult()} + deps := testImportDependencies(temp, exporter, media, executor) + for attempt := 0; attempt < 2; attempt++ { + cmd := newImportCommand(io.Discard, io.Discard, deps) + cmd.SilenceUsage = true + cmd.SetArgs([]string{"--from", "libtv", "--url", testLibTVURL}) + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "blocked") { + t.Fatalf("attempt %d error = %v, want blocked checkpoint", attempt+1, err) + } + } + if media.uploads != 1 || executor.calls != 0 { + t.Fatalf("upload/execute = %d/%d, want no blind retry or execute", media.uploads, executor.calls) + } +} + +func TestMediaCheckpointBlocksResumeAfterUploadCrashWindow(t *testing.T) { + opts := testMediaResolutionOptions(t) + crashing := &panickingImportMediaAPI{} + panicked := false + func() { + defer func() { + panicked = recover() != nil + }() + _, _ = resolveImportMedia(context.Background(), opts, crashing, io.Discard) + }() + if !panicked || crashing.uploads != 1 { + t.Fatalf("panic/uploads = %v/%d, want simulated crash after one dispatch", panicked, crashing.uploads) + } + checkpoint := readTestMediaCheckpoint(t, opts.CheckpointPath) + if len(checkpoint.Entries) != 1 || checkpoint.Entries[0].Status != mediaStatusUploadRequested { + t.Fatalf("checkpoint entries = %#v, want durable upload-requested", checkpoint.Entries) + } + + retry := &fakeImportMediaAPI{} + _, err := resolveImportMedia(context.Background(), opts, retry, io.Discard) + if err == nil || !strings.Contains(err.Error(), "blocked-on-interruption") { + t.Fatalf("resume error = %v, want blocked-on-interruption", err) + } + if retry.uploads != 0 { + t.Fatalf("resume uploads = %d, want no blind re-upload", retry.uploads) + } + checkpoint = readTestMediaCheckpoint(t, opts.CheckpointPath) + if checkpoint.Entries[0].Status != mediaStatusBlockedInterruption { + t.Fatalf("checkpoint status = %q, want blocked-on-interruption", checkpoint.Entries[0].Status) + } +} + +func TestMediaCheckpointDoesNotMarkMissingAKAsUploadRequested(t *testing.T) { + opts := testMediaResolutionOptions(t) + api := &missingAKPreflightMediaAPI{} + _, err := resolveImportMedia(context.Background(), opts, api, io.Discard) + if err == nil || !strings.Contains(err.Error(), "authentication failed") { + t.Fatalf("resolveImportMedia() error = %v, want explicit authentication failure", err) + } + if api.uploads != 0 { + t.Fatalf("uploads = %d, want preflight rejection before dispatch", api.uploads) + } + checkpoint := readTestMediaCheckpoint(t, opts.CheckpointPath) + if len(checkpoint.Entries) != 0 { + t.Fatalf("checkpoint entries = %#v, want no ambiguous upload marker", checkpoint.Entries) + } +} + +func TestDefaultJournalPathSeparatesPippitAccessKeys(t *testing.T) { + configDirectory := t.TempDir() + configDir := func() (string, error) { return configDirectory, nil } + source := canvasplan.Source{ + Provider: "libtv", ProjectID: "037a5c49e1b344e5adbc899ad93fdca9", Fingerprint: "sha256:" + strings.Repeat("1", 64), + } + target := "https://xyq.jianying.com|ppe_cli_canvas_ak" + firstScope := canvasImportAuthScope(&common.Runner{Config: &config.Config{AccessKey: "first-account-ak"}}) + secondScope := canvasImportAuthScope(&common.Runner{Config: &config.Config{AccessKey: "second-account-ak"}}) + firstPath, err := resolveImportJournalPath("", source, target, firstScope, configDir) + if err != nil { + t.Fatal(err) + } + secondPath, err := resolveImportJournalPath("", source, target, secondScope, configDir) + if err != nil { + t.Fatal(err) + } + if firstPath == secondPath { + t.Fatalf("journal paths are equal across Access Keys: %s", firstPath) + } + for _, secret := range []string{"first-account-ak", "second-account-ak", firstScope, secondScope} { + if strings.Contains(firstPath, secret) || strings.Contains(secondPath, secret) { + t.Fatalf("journal path persisted Access Key material: %q", secret) + } + } +} + +func TestReadyMediaCheckpointMustBelongToCurrentPippitAccount(t *testing.T) { + opts := testMediaResolutionOptions(t) + item := opts.Media[0] + checkpoint := &mediaCheckpoint{ + Schema: mediaCheckpointSchema, + Source: opts.Plan.Source, + Target: opts.Target, + BundleDirs: []string{opts.BundleDir}, + Entries: []mediaCheckpointEntry{{ + LogicalID: item.LogicalID, MediaType: item.MediaType, SHA256: item.SHA256, + Status: mediaStatusReady, AssetID: "asset-old-account", PippitAssetID: "pippit-old-account", + }}, + } + if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { + t.Fatal(err) + } + api := &fakeImportMediaAPI{queryErr: errors.New("asset not found for current account")} + _, err := resolveImportMedia(context.Background(), opts, api, io.Discard) + if err == nil || !strings.Contains(err.Error(), "current Pippit account") { + t.Fatalf("resolveImportMedia() error = %v, want cross-account checkpoint rejection", err) + } + if api.queries != 1 || api.uploads != 0 { + t.Fatalf("queries/uploads = %d/%d, want 1/0", api.queries, api.uploads) + } +} + +func TestMediaCheckpointLockPreventsConcurrentUpload(t *testing.T) { + opts := testMediaResolutionOptions(t) + first := &blockingImportMediaAPI{started: make(chan struct{}), release: make(chan struct{})} + firstDone := make(chan error, 1) + go func() { + _, err := resolveImportMedia(context.Background(), opts, first, io.Discard) + firstDone <- err + }() + <-first.started + + second := &fakeImportMediaAPI{} + _, secondErr := resolveImportMedia(context.Background(), opts, second, io.Discard) + if secondErr == nil || !strings.Contains(secondErr.Error(), "locked by another process") { + t.Fatalf("concurrent resolve error = %v, want checkpoint lock rejection", secondErr) + } + if second.uploads != 0 { + t.Fatalf("concurrent uploads = %d, want none", second.uploads) + } + close(first.release) + if err := <-firstDone; err != nil { + t.Fatalf("first resolve error = %v", err) + } + if first.uploads != 1 { + t.Fatalf("first uploads = %d, want one", first.uploads) + } +} + +func TestMediaCheckpointRejectsLockSymlink(t *testing.T) { + opts := testMediaResolutionOptions(t) + target := filepath.Join(t.TempDir(), "lock-target") + if err := os.WriteFile(target, []byte("target"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(opts.CheckpointPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, opts.CheckpointPath+".lock"); err != nil { + t.Fatal(err) + } + _, err := resolveImportMedia(context.Background(), opts, &fakeImportMediaAPI{}, io.Discard) + if err == nil || !strings.Contains(err.Error(), "must not be a symbolic link") { + t.Fatalf("resolveImportMedia() error = %v, want lock symlink rejection", err) + } +} + +func TestExporterEnvironmentUsesRuntimeAllowlist(t *testing.T) { + got := strings.Join(sanitizedExporterEnv([]string{ + "PATH=/bin", "HOME=/user/home", "LANG=zh_CN.UTF-8", "XDG_CONFIG_HOME=/user/config", + "LIBTV_CLI_PATH=/trusted/libtv", "PIPPIT_CLI_LIBTV_CACHE_DIR=/user/cache/libtv", + "HTTPS_PROXY=http://proxy.example:8080", "ALL_PROXY=socks5h://proxy.example:1080", "NO_PROXY=localhost,127.0.0.1", + "XYQ_ACCESS_KEY=secret-xyq", "PIPPIT_ACCESS_KEY=secret-pippit", "PIPPIT_CLI_PPE_ENV=ppe_lane", + "THIRD_PARTY_API_KEY=secret-third-party", "SSH_AUTH_SOCK=/private/ssh-agent", + "NODE_OPTIONS=--require=/private/injected.js", "PIPPIT_CLI_PACKAGE_ROOT=/pkg", + "HTTP_PROXY=http://proxy-user:proxy-password@proxy.example:8080", + "HTTPS_PROXY=file:///private/proxy", "HTTPS_PROXY=http://proxy.example:8080?token=secret-query", + "HTTPS_PROXY=http://proxy.example:8080#secret-fragment", "XDG_API_TOKEN=secret-xdg", + }), "\n") + for _, allowed := range []string{ + "PATH=/bin", "HOME=/user/home", "LANG=zh_CN.UTF-8", "XDG_CONFIG_HOME=/user/config", + "LIBTV_CLI_PATH=/trusted/libtv", "PIPPIT_CLI_LIBTV_CACHE_DIR=/user/cache/libtv", + "HTTPS_PROXY=http://proxy.example:8080", "ALL_PROXY=socks5h://proxy.example:1080", "NO_PROXY=localhost,127.0.0.1", + } { + if !strings.Contains(got, allowed) { + t.Fatalf("sanitized environment removed allowed runtime value %q: %s", allowed, got) + } + } + for _, forbidden := range []string{ + "secret-xyq", "secret-pippit", "ppe_lane", "secret-third-party", "/private/ssh-agent", + "--require=/private/injected.js", "PIPPIT_CLI_PACKAGE_ROOT=/pkg", "proxy-password", + "file:///private/proxy", "secret-query", "secret-fragment", "secret-xdg", + } { + if strings.Contains(got, forbidden) { + t.Fatalf("sanitized environment leaked forbidden value %q: %s", forbidden, got) + } + } +} + +func TestLibTVURLRejectsAmbiguousProjectIdentity(t *testing.T) { + _, err := normalizeLibTVURL( + "https://www.liblib.tv/canvas?projectId=037a5c49e1b344e5adbc899ad93fdca9&projectId=11111111111111111111111111111111", + ) + if err == nil { + t.Fatal("normalizeLibTVURL() error = nil, want duplicate projectId rejection") + } +} + +func TestTrustedCanvasURLUsesRootAndOverviewIdentifiers(t *testing.T) { + result := verifiedImportResult() + if err := validateTrustedCanvasURL(result); err != nil { + t.Fatalf("validateTrustedCanvasURL() error = %v", err) + } + result.WebURL = "https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=300" + if err := validateTrustedCanvasURL(result); err == nil { + t.Fatal("validateTrustedCanvasURL() error = nil, want overview-as-canvas rejection") + } + result.WebURL = "https://xyq.jianying.com:8443/novel/detail/canvas?projectId=100" + if err := validateTrustedCanvasURL(result); err == nil { + t.Fatal("validateTrustedCanvasURL() error = nil, want untrusted port rejection") + } +} + +const testLibTVURL = "https://www.liblib.tv/canvas?spaceId=3872811&projectId=037a5c49e1b344e5adbc899ad93fdca9" + +func testImportDependencies( + root string, + exporter importExporter, + media importMediaAPI, + executor importExecutor, +) importDependencies { + return importDependencies{ + exporter: exporter, + media: media, + executor: executor, + openURL: func(context.Context, string) error { return nil }, + userCacheDir: func() (string, error) { return filepath.Join(root, "cache"), nil }, + userConfigDir: func() (string, error) { return filepath.Join(root, "config"), nil }, + target: func() string { return "https://xyq.jianying.com|ppe_cli_canvas_ak" }, + authScope: func() string { return strings.Repeat("a", 64) }, + } +} + +func testImportPlan(t *testing.T, degraded bool) (canvasplan.Plan, map[string][]byte) { + t.Helper() + payload := []byte("same-image-bytes") + digest := sha256.Sum256(payload) + hash := hex.EncodeToString(digest[:]) + size := int64(len(payload)) + plan := canvasplan.Plan{ + Schema: canvasplan.PlanSchema, + Title: "Imported LibTV canvas", + Source: canvasplan.Source{Provider: "libtv", ProjectID: "037a5c49e1b344e5adbc899ad93fdca9", Fingerprint: "sha256:" + strings.Repeat("1", 64)}, + RequiredMedia: []canvasplan.MediaRequirement{ + {LogicalID: "media:image-1", SourceNodeID: "image-1", FileName: "one.png", MediaType: "image", LocalPath: "media/one.png", SHA256: hash, Metadata: canvasplan.MediaMetadata{ByteSize: &size}}, + {LogicalID: "media:image-2", SourceNodeID: "image-2", FileName: "two.png", MediaType: "image", LocalPath: "media/two.png", SHA256: hash, Metadata: canvasplan.MediaMetadata{ByteSize: &size}}, + }, + Nodes: []canvasplan.Node{ + {LogicalID: "node:image-1", SourceNodeID: "image-1", Title: "One", Position: canvasplan.Position{X: 0, Y: 0}, Size: canvasplan.Size{Width: 100, Height: 100}, Kind: "image", TargetType: "biz/image", MediaLogicalID: "media:image-1"}, + {LogicalID: "node:image-2", SourceNodeID: "image-2", Title: "Two", Position: canvasplan.Position{X: 120, Y: 0}, Size: canvasplan.Size{Width: 100, Height: 100}, Kind: "image", TargetType: "biz/image", MediaLogicalID: "media:image-2"}, + }, + Groups: []canvasplan.Group{}, + Edges: []canvasplan.Edge{}, + } + if degraded { + plan.Degradations = []json.RawMessage{json.RawMessage(`{"code":"test.degradation"}`)} + } + return plan, map[string][]byte{"media/one.png": payload, "media/two.png": payload} +} + +func testSingleMediaPlan(t *testing.T) (canvasplan.Plan, map[string][]byte) { + plan, media := testImportPlan(t, false) + plan.RequiredMedia = plan.RequiredMedia[:1] + plan.Nodes = plan.Nodes[:1] + delete(media, "media/two.png") + return plan, media +} + +func testMediaResolutionOptions(t *testing.T) mediaResolutionOptions { + t.Helper() + root := t.TempDir() + plan, mediaBytes := testSingleMediaPlan(t) + bundleRoot := filepath.Join(root, "cache", "exports") + bundleDir := filepath.Join(bundleRoot, "export-fixture") + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + if _, err := exporter.Export(context.Background(), testLibTVURL, bundleDir, io.Discard); err != nil { + t.Fatalf("prepare export fixture: %v", err) + } + media, err := readAndValidateExportMedia(bundleDir, plan) + if err != nil { + t.Fatalf("validate export fixture media: %v", err) + } + return mediaResolutionOptions{ + Plan: plan, + Media: media, + Target: "https://xyq.jianying.com|ppe_cli_canvas_ak", + BundleDir: bundleDir, + BundleRoot: bundleRoot, + CanvasJournalPath: filepath.Join(root, "state", "canvas.journal.json"), + CheckpointPath: filepath.Join(root, "state", "canvas.journal.json.media.json"), + } +} + +func readTestMediaCheckpoint(t *testing.T, path string) mediaCheckpoint { + t.Helper() + payload, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read media checkpoint: %v", err) + } + var checkpoint mediaCheckpoint + if err := json.Unmarshal(payload, &checkpoint); err != nil { + t.Fatalf("decode media checkpoint: %v", err) + } + return checkpoint +} + +func verifiedImportResult() *canvasplan.ExecutionResult { + return &canvasplan.ExecutionResult{ + State: canvasplan.StateVerified, + ProjectID: "100", + RootCanvasID: "200", + OverviewPippitAssetID: "300", + WebURL: "https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200&overviewPippitAssetId=300", + Verification: &canvasplan.Verification{Verified: true}, + } +} + +func writeTestJSON(path string, value any) error { + payload, err := json.Marshal(value) + if err != nil { + return err + } + return os.WriteFile(path, payload, 0o600) +} From aced62fadb9c95e9c8f7fbb89a7fe482ea1eda87 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:48:03 +0800 Subject: [PATCH 16/48] feat: make LibTV plans executable Co-authored-by: Codex <codex@openai.com> --- adapters/libtv/plan.mjs | 100 +++++++++++++++++--- adapters/libtv/plan.test.mjs | 64 +++++++++++-- adapters/libtv/testdata/media-manifest.json | 2 + 3 files changed, 148 insertions(+), 18 deletions(-) diff --git a/adapters/libtv/plan.mjs b/adapters/libtv/plan.mjs index 6c65140..5e17136 100644 --- a/adapters/libtv/plan.mjs +++ b/adapters/libtv/plan.mjs @@ -2,20 +2,24 @@ import { createHash } from 'node:crypto'; const PLAN_SCHEMA = 'pippit-canvas-plan/0.1'; const SNAPSHOT_SCHEMA = 'xyq-libtv-snapshot/0.1'; -const SUPPORTED_NODE_TYPES = new Set(['group', 'video', 'audio', 'video-clip']); +const SUPPORTED_NODE_TYPES = new Set(['group', 'image', 'video', 'audio', 'video-clip']); -function canonicalize(value) { - if (Array.isArray(value)) return value.map(canonicalize); - if (!value || typeof value !== 'object') return value; +function canonicalize(value, replacer, key = '') { + const replaced = replacer ? replacer(value, key) : value; + if (Array.isArray(replaced)) { + return replaced.map((child) => canonicalize(child, replacer)).filter((child) => child !== undefined); + } + if (!replaced || typeof replaced !== 'object') return replaced; return Object.fromEntries( - Object.keys(value) + Object.keys(replaced) .sort() - .map((key) => [key, canonicalize(value[key])]), + .map((childKey) => [childKey, canonicalize(replaced[childKey], replacer, childKey)]) + .filter(([, child]) => child !== undefined), ); } -function sha256Json(value) { - return createHash('sha256').update(JSON.stringify(canonicalize(value))).digest('hex'); +function sha256Json(value, replacer) { + return createHash('sha256').update(JSON.stringify(canonicalize(value, replacer))).digest('hex'); } function nonEmptyString(value) { @@ -55,13 +59,41 @@ function mediaManifestByNodeId(mediaManifest) { const sourceNodeId = nonEmptyString(item?.sourceNodeId ?? item?.source_node_id); if (!sourceNodeId || result.has(sourceNodeId)) continue; result.set(sourceNodeId, { + byteSize: normalizeByteSize(item?.byteSize ?? item?.byte_size), fileName: nonEmptyString(item?.fileName ?? item?.file_name ?? item?.path), + localPath: normalizeLocalMediaPath(item?.localPath ?? item?.local_path ?? item?.relativePath ?? item?.relative_path), + mediaType: nonEmptyString(item?.mediaType ?? item?.media_type), + sha256: normalizeSHA256(item?.sha256), url: normalizeHTTPSURL(item?.url), }); } return result; } +function normalizeByteSize(value) { + if (value === undefined || value === null || value === '') return undefined; + const number = Number(value); + if (!Number.isSafeInteger(number) || number <= 0) throw new Error('media byte_size must be a positive safe integer'); + return number; +} + +function normalizeSHA256(value) { + const text = nonEmptyString(value); + if (!text) return undefined; + const hex = text.startsWith('sha256:') ? text.slice(7) : text; + if (!/^[0-9a-f]{64}$/i.test(hex)) throw new Error('media sha256 must contain 64 hexadecimal characters'); + return hex.toLowerCase(); +} + +function normalizeLocalMediaPath(value) { + const text = nonEmptyString(value); + if (!text) return undefined; + if (text.includes('\\') || text.startsWith('/') || text.split('/').some((part) => !part || part === '.' || part === '..')) { + throw new Error(`invalid local media path: ${text.slice(0, 120)}`); + } + return text; +} + function assetReferenceByNodeId(snapshot) { const result = new Map(); for (const item of snapshot.assetReferences ?? []) { @@ -119,7 +151,7 @@ function mediaMetadata(data) { } function fallbackFileName(node, metadata) { - const extension = metadata.extension ?? (node.type === 'audio' ? 'audio' : 'video'); + const extension = metadata.extension ?? node.type; const stem = (nonEmptyString(node.name) ?? node.id) .replaceAll(/[\\/:*?"<>|]/g, '_') .slice(0, 120); @@ -212,7 +244,23 @@ function sourceFingerprint(snapshot) { project: snapshot.project, nodeDetails: snapshot.nodeDetails ?? [], assetReferences: snapshot.assetReferences ?? [], - })}`; + }, fingerprintReplacer)}`; +} + +function fingerprintReplacer(value, key) { + if (key && /token|cookie|authorization|credential|signature|secret|access.?key|expires?/i.test(key)) { + return undefined; + } + if (typeof value !== 'string') return value; + try { + const url = new URL(value); + if (url.protocol === 'http:' || url.protocol === 'https:') { + return `${url.protocol}//${url.host}${url.pathname}`; + } + } catch { + // Non-URL strings are stable source data. + } + return value; } function canvasTitle(snapshot, projectId, override) { @@ -289,22 +337,50 @@ function convertSnapshotToCanvasPlan(snapshot, options = {}) { const mediaType = sourceNode.type; const manifestItem = manifest.get(sourceNodeId) ?? {}; - const url = manifestItem.url ?? assetReferences.get(sourceNodeId) ?? normalizeHTTPSURL(detail?.url); + if (manifestItem.mediaType && manifestItem.mediaType !== mediaType) { + throw new Error(`media manifest type ${manifestItem.mediaType} does not match ${mediaType} node ${sourceNodeId}`); + } + if (Boolean(manifestItem.localPath) !== Boolean(manifestItem.sha256)) { + throw new Error(`media manifest node ${sourceNodeId} must provide local_path and sha256 together`); + } + const url = manifestItem.localPath + ? undefined + : manifestItem.url ?? assetReferences.get(sourceNodeId) ?? normalizeHTTPSURL(detail?.url); const metadata = mediaMetadata(detail); + if (manifestItem.byteSize) metadata.byte_size = manifestItem.byteSize; const fileName = safeFileName(manifestItem.fileName) ?? safeFileName(fileNameFromURL(url)) ?? fallbackFileName(sourceNode, metadata); const mediaLogicalId = logicalMediaId(sourceNodeId); + if (!manifestItem.localPath && !url) { + if (mediaType !== 'audio') { + const placeholderKind = `${mediaType}-placeholder`; + nodes.push({ + ...base, + kind: placeholderKind, + target_type: `biz/${mediaType}`, + }); + degradations.push({ + code: 'libtv.media.empty_placeholder', + source_node_id: sourceNodeId, + message: `LibTV ${mediaType} node has no downloadable result and will become an empty ${mediaType} placeholder.`, + }); + return; + } + // Snapshot-only compatibility: legacy callers resolve audio out of band. + } requiredMedia.push(compact({ logical_id: mediaLogicalId, source_node_id: sourceNodeId, file_name: fileName, media_type: mediaType, + local_path: manifestItem.localPath, + sha256: manifestItem.sha256, url, metadata, })); nodes.push({ ...base, kind: mediaType, - target_type: mediaType === 'audio' ? 'biz/audio' : 'biz/video', + target_type: `biz/${mediaType}`, media_logical_id: mediaLogicalId, }); }); diff --git a/adapters/libtv/plan.test.mjs b/adapters/libtv/plan.test.mjs index 5b4e780..04ce8ba 100644 --- a/adapters/libtv/plan.test.mjs +++ b/adapters/libtv/plan.test.mjs @@ -23,7 +23,7 @@ async function testRepresentativeSnapshot() { assert.deepEqual(plan.source, { provider: 'libtv', project_id: 'fixture-project', - fingerprint: 'sha256:c33d2d9580d2a3a57fd06cac45e44f8881cdd70df6aeffc178e0f6077c11cf68', + fingerprint: 'sha256:db75e738a45ec45c6753532f1e10bcb89f9aee7370c8c24f0b13e51368e9f2a3', }); assert.equal(plan.title, 'LibTV adapter fixture'); assert.equal(plan.required_media.length, 2); @@ -43,6 +43,8 @@ async function testRepresentativeSnapshot() { }, }); assert.equal(plan.required_media[1].file_name, 'input.wav'); + assert.equal(plan.required_media[1].local_path, 'media/input.wav'); + assert.equal(plan.required_media[1].sha256, 'a'.repeat(64)); assert.equal(plan.required_media[1].url, undefined); assert.equal(plan.nodes.length, 3); assert.deepEqual(plan.nodes[2].input_node_logical_ids, ['node:video-1']); @@ -67,14 +69,57 @@ async function testRepresentativeSnapshot() { async function testDeterminismAndVolatileExportTime() { const snapshot = await readJson(fixtureURL); - const first = convertSnapshotToCanvasPlan(snapshot); - const second = convertSnapshotToCanvasPlan({ ...snapshot, exportedAt: '2099-01-01T00:00:00.000Z' }); + const mediaManifest = await readJson(manifestURL); + const first = convertSnapshotToCanvasPlan(snapshot, { mediaManifest }); + const second = convertSnapshotToCanvasPlan({ ...snapshot, exportedAt: '2099-01-01T00:00:00.000Z' }, { mediaManifest }); assert.deepEqual(first, second); - assert.equal(convertSnapshotToCanvasPlan(snapshot, { title: 'Override title' }).title, 'Override title'); + assert.equal(convertSnapshotToCanvasPlan(snapshot, { mediaManifest, title: 'Override title' }).title, 'Override title'); + + const rotated = clone(snapshot); + rotated.nodeDetails[0].detail.data.url[0] = 'https://media.example.test/input.mp4?token=rotated&expires=999'; + rotated.assetReferences[0].url = 'https://media.example.test/input.mp4?signature=another'; + assert.equal(convertSnapshotToCanvasPlan(rotated, { mediaManifest }).source.fingerprint, first.source.fingerprint); +} + +async function testImageAndEmptyPlaceholder() { + const snapshot = await readJson(fixtureURL); + snapshot.project.nodes.push({ + id: 'image-1', + name: 'Pending image', + type: 'image', + position: { x: 1500, y: 20 }, + width: 320, + height: 320, + }); + snapshot.nodeDetails.push({ sourceNodeId: 'image-1', detail: { data: { type: 'image', url: [] } } }); + const mediaManifest = await readJson(manifestURL); + const placeholder = convertSnapshotToCanvasPlan(snapshot, { mediaManifest }); + assert.equal(placeholder.nodes.find((node) => node.source_node_id === 'image-1').kind, 'image-placeholder'); + assert.equal(placeholder.degradations.some((item) => item.source_node_id === 'image-1'), true); + + mediaManifest.uploads.push({ + sourceNodeId: 'image-1', + fileName: 'image.png', + mediaType: 'image', + relative_path: 'media/image.png', + sha256: 'B'.repeat(64), + }); + const imported = convertSnapshotToCanvasPlan(snapshot, { mediaManifest }); + assert.equal(imported.nodes.find((node) => node.source_node_id === 'image-1').kind, 'image'); + assert.deepEqual(imported.required_media.find((item) => item.source_node_id === 'image-1'), { + logical_id: 'media:image-1', + source_node_id: 'image-1', + file_name: 'image.png', + media_type: 'image', + local_path: 'media/image.png', + sha256: 'b'.repeat(64), + metadata: {}, + }); } async function testRejectsUnsafeOrInvalidInputs() { const snapshot = await readJson(fixtureURL); + const mediaManifest = await readJson(manifestURL); const unsafe = clone(snapshot); unsafe.nodeDetails[0].detail.data.url = ['http://media.example.test/input.mp4']; unsafe.assetReferences = []; @@ -88,15 +133,22 @@ async function testRejectsUnsafeOrInvalidInputs() { unsupported.project.nodes[1].type = 'prompt'; assert.throws(() => convertSnapshotToCanvasPlan(unsupported), /unsupported LibTV node type/); - assert.throws(() => convertSnapshotToCanvasPlan(snapshot, { title: 'x'.repeat(51) }), /must not exceed 50/); + assert.throws(() => convertSnapshotToCanvasPlan(snapshot, { mediaManifest, title: 'x'.repeat(51) }), /must not exceed 50/); + + const badManifest = await readJson(manifestURL); + badManifest.uploads[1].relative_path = '../input.wav'; + assert.throws(() => convertSnapshotToCanvasPlan(snapshot, { mediaManifest: badManifest }), /invalid local media path/); const encodedSlash = clone(snapshot); encodedSlash.assetReferences[0].url = 'https://media.example.test/nested%2Fevil.mp4'; encodedSlash.nodeDetails[0].detail.data.url = []; - assert.equal(convertSnapshotToCanvasPlan(encodedSlash).required_media[0].file_name, 'evil.mp4'); + const audioOnlyManifest = clone(mediaManifest); + audioOnlyManifest.uploads = audioOnlyManifest.uploads.filter((item) => item.sourceNodeId === 'audio-1'); + assert.equal(convertSnapshotToCanvasPlan(encodedSlash, { mediaManifest: audioOnlyManifest }).required_media[0].file_name, 'evil.mp4'); } await testRepresentativeSnapshot(); await testDeterminismAndVolatileExportTime(); +await testImageAndEmptyPlaceholder(); await testRejectsUnsafeOrInvalidInputs(); process.stdout.write('libtv plan adapter tests passed\n'); diff --git a/adapters/libtv/testdata/media-manifest.json b/adapters/libtv/testdata/media-manifest.json index 32d5394..7d87d0a 100644 --- a/adapters/libtv/testdata/media-manifest.json +++ b/adapters/libtv/testdata/media-manifest.json @@ -14,6 +14,8 @@ { "sourceNodeId": "audio-1", "fileName": "input.wav", + "relative_path": "media/input.wav", + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "assetId": "must-not-leak", "pippitAssetId": "must-not-leak" } From 197008148ddbe9bc23454d9538081381d8553d84 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:48:16 +0800 Subject: [PATCH 17/48] feat: verify official LibTV CLI downloads Co-authored-by: Codex <codex@openai.com> --- adapters/libtv/bootstrap.mjs | 325 ++++++++++++++++++++++++++++++ adapters/libtv/bootstrap.test.mjs | 150 ++++++++++++++ 2 files changed, 475 insertions(+) create mode 100644 adapters/libtv/bootstrap.mjs create mode 100644 adapters/libtv/bootstrap.test.mjs diff --git a/adapters/libtv/bootstrap.mjs b/adapters/libtv/bootstrap.mjs new file mode 100644 index 0000000..54dbafa --- /dev/null +++ b/adapters/libtv/bootstrap.mjs @@ -0,0 +1,325 @@ +import { createHash } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { createReadStream, createWriteStream } from 'node:fs'; +import { + chmod, + copyFile, + lstat, + mkdir, + mkdtemp, + readdir, + rename, + rm, + writeFile, +} from 'node:fs/promises'; +import { get as httpsGet } from 'node:https'; +import { homedir, tmpdir } from 'node:os'; +import { basename, join, parse, resolve } from 'node:path'; +import { Transform } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; + +const OFFICIAL_CLI_VERSION = '1.1.3'; +const OFFICIAL_CLI_ORIGIN = 'https://liblibai-web-static.liblib.cloud'; +const OFFICIAL_INSTALLERS = Object.freeze({ + shell: `${OFFICIAL_CLI_ORIGIN}/cli/1.1.3/install-libtv-cli.sh`, + powershell: `${OFFICIAL_CLI_ORIGIN}/cli/1.1.3/install-libtv-cli.ps1`, + cmd: `${OFFICIAL_CLI_ORIGIN}/cli/1.1.3/install-libtv-cli.bat`, +}); +// Activity 240 version 1.1.3 artifacts, independently downloaded and hashed on 2026-08-11. +const OFFICIAL_ARTIFACTS = Object.freeze({ + 'darwin-arm64': Object.freeze({ + zipName: 'libtv-macos-arm64.zip', + zipSHA256: '95c21012530917da8ce69eb01ebb197f418783904a0dcd16ddcf27efe5139df7', + binarySHA256: '1abc924df7fb3d3428b890909c78241749136797d3c046c0b810653e9fbf6fdd', + executable: 'libtv', + }), + 'darwin-x64': Object.freeze({ + zipName: 'libtv-macos-x64.zip', + zipSHA256: 'e8dfad868919522cd1e3ea5f437506bdd193684b96cf3db89512130abddb6347', + binarySHA256: '0248dd94bdbee377f67153883110f6d48e2c1bebeeeb69d1673c30e257c84592', + executable: 'libtv', + }), + 'linux-arm64': Object.freeze({ + zipName: 'libtv-linux-arm64.zip', + zipSHA256: '369b43f5be1d28dbbde7c1b6711ed746bf9bff1028ba794a6dae4fa01bed601c', + binarySHA256: '1fe47f1d3b56f826e72c4d4a9b452a1538f5d6974d7ee319678685387d14f43f', + executable: 'libtv', + }), + 'linux-x64': Object.freeze({ + zipName: 'libtv-linux-x64.zip', + zipSHA256: 'cf86f462c5aed60f95dca978cc91ece98c60bcfa27337da008ad59953c3ea7da', + binarySHA256: 'e79ad52170556b44e957174f880c8a69057f668ddd0b9a4011524cac072c31f3', + executable: 'libtv', + }), + 'win32-arm64': Object.freeze({ + zipName: 'libtv-windows-arm64.zip', + zipSHA256: 'a64f14987ba44cf7345451d557a4dfe527db7b343a1c7032ecf23bcc320974ee', + binarySHA256: '3ccff728c39277d8ad596d8a3b24bbc071579f83f6bc102c4051233cab2734bc', + executable: 'libtv.exe', + }), + 'win32-x64': Object.freeze({ + // The official PowerShell installer names this remote architecture "amd64". + zipName: 'libtv-windows-amd64.zip', + zipSHA256: '5c5e14b683ebbafba4b2c156be305384bd9f558169d019442e53b2ea04206bd5', + binarySHA256: 'a607ea1f557cb513f302138e64d86312b9dfa9e7eed9dffc5c07f5559192f3fb', + executable: 'libtv.exe', + }), +}); +const MAX_ZIP_BYTES = 256 << 20; + +class LibTVBootstrapError extends Error { + constructor(code, message) { + super(message); + this.name = 'LibTVBootstrapError'; + this.code = code; + } +} + +function artifactFor(platform = process.platform, arch = process.arch, artifacts = OFFICIAL_ARTIFACTS) { + const key = `${platform}-${arch}`; + const artifact = artifacts[key]; + if (!artifact) throw new LibTVBootstrapError('UNSUPPORTED_PLATFORM', `LibTV CLI bootstrap does not support ${key}`); + return { + ...artifact, + key, + url: `${OFFICIAL_CLI_ORIGIN}/cli/${OFFICIAL_CLI_VERSION}/${artifact.zipName}`, + }; +} + +function defaultCacheRoot(environment = process.env, platform = process.platform) { + if (environment.PIPPIT_CLI_LIBTV_CACHE_DIR) return resolve(environment.PIPPIT_CLI_LIBTV_CACHE_DIR); + const home = environment.HOME || environment.USERPROFILE || homedir(); + let base; + if (platform === 'darwin') base = join(home, 'Library', 'Caches'); + else if (platform === 'win32') base = environment.LOCALAPPDATA || join(home, 'AppData', 'Local'); + else base = environment.XDG_CACHE_HOME || join(home, '.cache'); + return join(base, 'pippit-cli', 'tools', 'libtv'); +} + +function assertSafeCacheRoot(cacheRoot, environment, platform) { + const home = resolve(environment.HOME || environment.USERPROFILE || homedir()); + const unsafe = new Set([parse(cacheRoot).root, home, resolve(tmpdir())]); + if (platform === 'darwin') unsafe.add(join(home, 'Library', 'Caches')); + else if (platform === 'win32') unsafe.add(resolve(environment.LOCALAPPDATA || join(home, 'AppData', 'Local'))); + else unsafe.add(resolve(environment.XDG_CACHE_HOME || join(home, '.cache'))); + if (unsafe.has(cacheRoot)) { + throw new LibTVBootstrapError('UNSAFE_CACHE_ROOT', 'refusing to use a broad directory as the LibTV CLI cache root'); + } +} + +async function fileSHA256(path) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest('hex'); +} + +async function verifiedCachedBinary(platformDirectory, artifact) { + const binaryPath = join(platformDirectory, artifact.executable); + let info; + try { + info = await lstat(binaryPath); + } catch (error) { + if (error?.code === 'ENOENT') { + if (await pathExists(platformDirectory)) { + throw new LibTVBootstrapError('CACHE_INTEGRITY_FAILED', 'LibTV CLI cache directory is incomplete'); + } + return undefined; + } + throw error; + } + if (!info.isFile() || info.isSymbolicLink()) { + throw new LibTVBootstrapError('CACHE_INTEGRITY_FAILED', 'cached LibTV CLI is not a regular file'); + } + const digest = await fileSHA256(binaryPath); + if (digest !== artifact.binarySHA256) { + throw new LibTVBootstrapError('CACHE_INTEGRITY_FAILED', 'cached LibTV CLI failed SHA-256 verification'); + } + await chmod(binaryPath, 0o700); + return binaryPath; +} + +async function pathExists(path) { + try { + await lstat(path); + return true; + } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + +function httpsResponse(url) { + return new Promise((resolveResponse, rejectResponse) => { + const parsed = new URL(url); + if (parsed.protocol !== 'https:' || parsed.origin !== OFFICIAL_CLI_ORIGIN) { + rejectResponse(new LibTVBootstrapError('UNTRUSTED_DOWNLOAD', 'refusing a non-official LibTV CLI URL')); + return; + } + const request = httpsGet(parsed, { headers: { 'user-agent': 'pippit-cli-libtv-bootstrap/0.1' } }, resolveResponse); + request.setTimeout(30_000, () => request.destroy(new Error('download timed out'))); + request.on('error', rejectResponse); + }); +} + +async function downloadOfficialZip({ url, destination }) { + const response = await httpsResponse(url); + if (response.statusCode !== 200) { + response.resume(); + throw new LibTVBootstrapError('DOWNLOAD_FAILED', `official LibTV CLI download returned HTTP ${response.statusCode}`); + } + const declared = Number(response.headers['content-length']); + if (Number.isFinite(declared) && declared > MAX_ZIP_BYTES) { + response.destroy(); + throw new LibTVBootstrapError('DOWNLOAD_TOO_LARGE', 'official LibTV CLI ZIP exceeds the size limit'); + } + let received = 0; + const limiter = new Transform({ + transform(chunk, _encoding, callback) { + received += chunk.length; + if (received > MAX_ZIP_BYTES) callback(new LibTVBootstrapError('DOWNLOAD_TOO_LARGE', 'official LibTV CLI ZIP exceeds the size limit')); + else callback(null, chunk); + }, + }); + await pipeline(response, limiter, createWriteStream(destination, { mode: 0o600 })); + await chmod(destination, 0o600); +} + +function runTool(command, args, environment) { + return new Promise((resolveResult) => { + let stdout = ''; + let stderr = ''; + let settled = false; + const child = spawn(command, args, { env: environment, shell: false, stdio: ['ignore', 'pipe', 'pipe'] }); + const finish = (result) => { + if (settled) return; + settled = true; + resolveResult({ stdout, stderr, ...result }); + }; + child.on('error', (error) => finish({ exitCode: null, error })); + child.stdout.on('data', (chunk) => { if (stdout.length < (4 << 20)) stdout += chunk.toString('utf8'); }); + child.stderr.on('data', (chunk) => { if (stderr.length < (4 << 20)) stderr += chunk.toString('utf8'); }); + child.on('close', (exitCode) => finish({ exitCode })); + }); +} + +function validateArchiveListing(listing) { + const entries = listing.split(/\r?\n/).filter(Boolean); + if (entries.length === 0) throw new LibTVBootstrapError('INVALID_ARCHIVE', 'official LibTV CLI ZIP is empty'); + for (const entry of entries) { + const parts = entry.split('/').filter(Boolean); + if (entry.includes('\\') || entry.startsWith('/') || /^[A-Za-z]:/.test(entry) || parts.includes('..')) { + throw new LibTVBootstrapError('UNSAFE_ARCHIVE', 'official LibTV CLI ZIP contains an unsafe path'); + } + } +} + +async function extractOfficialZip({ zipPath, destination, platform, environment }) { + const command = platform === 'win32' ? 'tar.exe' : 'unzip'; + const listArgs = platform === 'win32' ? ['-tf', zipPath] : ['-Z1', zipPath]; + const listing = await runTool(command, listArgs, environment); + if (listing.exitCode !== 0) { + const hint = platform === 'win32' ? 'Windows 10+ built-in tar.exe is required' : 'unzip is required'; + throw new LibTVBootstrapError('EXTRACTOR_UNAVAILABLE', `cannot inspect official LibTV CLI ZIP; ${hint}`); + } + validateArchiveListing(listing.stdout); + const extractArgs = platform === 'win32' + ? ['-xf', zipPath, '-C', destination] + : ['-q', zipPath, '-d', destination]; + const extracted = await runTool(command, extractArgs, environment); + if (extracted.exitCode !== 0) { + throw new LibTVBootstrapError('EXTRACT_FAILED', 'failed to extract verified official LibTV CLI ZIP'); + } +} + +async function findBinary(root, executable, current = root) { + const candidates = []; + for (const entry of await readdir(current, { withFileTypes: true })) { + const path = join(current, entry.name); + if (entry.isSymbolicLink()) throw new LibTVBootstrapError('UNSAFE_ARCHIVE', 'official LibTV CLI ZIP contains a symbolic link'); + if (entry.isDirectory()) candidates.push(...await findBinary(root, executable, path)); + else if (entry.isFile() && basename(path) === executable) candidates.push(path); + } + return candidates; +} + +async function writeMetadata(path, artifact) { + const metadata = { + schema: 'pippit-libtv-tool-cache/0.1', + version: OFFICIAL_CLI_VERSION, + platform: artifact.key, + source: artifact.url, + zip_sha256: artifact.zipSHA256, + binary_sha256: artifact.binarySHA256, + }; + await writeFile(path, `${JSON.stringify(metadata, null, 2)}\n`, { mode: 0o600 }); + await chmod(path, 0o600); +} + +async function bootstrapLibTVCLI(options = {}) { + const platform = options.platform ?? process.platform; + const arch = options.arch ?? process.arch; + const artifact = artifactFor(platform, arch, options.artifacts); + const sourceEnvironment = options.env ?? process.env; + const environment = options.environment ?? sourceEnvironment; + const cacheRoot = resolve(options.cacheRoot ?? defaultCacheRoot(sourceEnvironment, platform)); + assertSafeCacheRoot(cacheRoot, sourceEnvironment, platform); + const versionDirectory = join(cacheRoot, OFFICIAL_CLI_VERSION); + const platformDirectory = join(versionDirectory, artifact.key); + const cached = await verifiedCachedBinary(platformDirectory, artifact); + if (cached) return cached; + + await mkdir(versionDirectory, { recursive: true, mode: 0o700 }); + await chmod(cacheRoot, 0o700); + await chmod(versionDirectory, 0o700); + const staging = await mkdtemp(join(versionDirectory, '.bootstrap-')); + await chmod(staging, 0o700); + try { + const zipPath = join(staging, artifact.zipName); + await (options.download ?? downloadOfficialZip)({ url: artifact.url, destination: zipPath, artifact }); + if (await fileSHA256(zipPath) !== artifact.zipSHA256) { + throw new LibTVBootstrapError('ZIP_INTEGRITY_FAILED', 'official LibTV CLI ZIP failed SHA-256 verification'); + } + const extractionDirectory = join(staging, 'extracted'); + await mkdir(extractionDirectory, { mode: 0o700 }); + await (options.extract ?? extractOfficialZip)({ + zipPath, + destination: extractionDirectory, + platform, + environment, + artifact, + }); + const binaries = await findBinary(extractionDirectory, artifact.executable); + if (binaries.length !== 1) { + throw new LibTVBootstrapError('INVALID_ARCHIVE', 'verified official LibTV CLI ZIP must contain exactly one binary'); + } + if (await fileSHA256(binaries[0]) !== artifact.binarySHA256) { + throw new LibTVBootstrapError('BINARY_INTEGRITY_FAILED', 'official LibTV CLI binary failed SHA-256 verification'); + } + const installed = join(staging, 'installed'); + await mkdir(installed, { mode: 0o700 }); + const installedBinary = join(installed, artifact.executable); + await copyFile(binaries[0], installedBinary); + await chmod(installedBinary, 0o700); + await writeMetadata(join(installed, 'metadata.json'), artifact); + try { + await rename(installed, platformDirectory); + } catch (error) { + if (!['EEXIST', 'ENOTEMPTY'].includes(error?.code)) throw error; + } + const verified = await verifiedCachedBinary(platformDirectory, artifact); + if (!verified) throw new LibTVBootstrapError('CACHE_INSTALL_FAILED', 'verified LibTV CLI cache install did not complete'); + return verified; + } finally { + await rm(staging, { recursive: true, force: true }); + } +} + +export { + LibTVBootstrapError, + OFFICIAL_ARTIFACTS, + OFFICIAL_CLI_VERSION, + OFFICIAL_INSTALLERS, + artifactFor, + bootstrapLibTVCLI, + defaultCacheRoot, +}; diff --git a/adapters/libtv/bootstrap.test.mjs b/adapters/libtv/bootstrap.test.mjs new file mode 100644 index 0000000..9433bdd --- /dev/null +++ b/adapters/libtv/bootstrap.test.mjs @@ -0,0 +1,150 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { chmod, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; + +import { + OFFICIAL_ARTIFACTS, + OFFICIAL_CLI_VERSION, + artifactFor, + bootstrapLibTVCLI, +} from './bootstrap.mjs'; + +const AUDITED_ARTIFACTS = { + 'darwin-arm64': ['libtv-macos-arm64.zip', '95c21012530917da8ce69eb01ebb197f418783904a0dcd16ddcf27efe5139df7', '1abc924df7fb3d3428b890909c78241749136797d3c046c0b810653e9fbf6fdd'], + 'darwin-x64': ['libtv-macos-x64.zip', 'e8dfad868919522cd1e3ea5f437506bdd193684b96cf3db89512130abddb6347', '0248dd94bdbee377f67153883110f6d48e2c1bebeeeb69d1673c30e257c84592'], + 'linux-arm64': ['libtv-linux-arm64.zip', '369b43f5be1d28dbbde7c1b6711ed746bf9bff1028ba794a6dae4fa01bed601c', '1fe47f1d3b56f826e72c4d4a9b452a1538f5d6974d7ee319678685387d14f43f'], + 'linux-x64': ['libtv-linux-x64.zip', 'cf86f462c5aed60f95dca978cc91ece98c60bcfa27337da008ad59953c3ea7da', 'e79ad52170556b44e957174f880c8a69057f668ddd0b9a4011524cac072c31f3'], + 'win32-arm64': ['libtv-windows-arm64.zip', 'a64f14987ba44cf7345451d557a4dfe527db7b343a1c7032ecf23bcc320974ee', '3ccff728c39277d8ad596d8a3b24bbc071579f83f6bc102c4051233cab2734bc'], + 'win32-x64': ['libtv-windows-amd64.zip', '5c5e14b683ebbafba4b2c156be305384bd9f558169d019442e53b2ea04206bd5', 'a607ea1f557cb513f302138e64d86312b9dfa9e7eed9dffc5c07f5559192f3fb'], +}; + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function fakeArtifact(zipBytes, binaryBytes) { + return { + 'linux-x64': { + zipName: 'libtv-linux-x64.zip', + zipSHA256: sha256(zipBytes), + binarySHA256: sha256(binaryBytes), + executable: 'libtv', + }, + }; +} + +async function testOfficialArtifactMatrix() { + assert.deepEqual(Object.keys(OFFICIAL_ARTIFACTS).sort(), Object.keys(AUDITED_ARTIFACTS).sort()); + for (const [key, expected] of Object.entries(OFFICIAL_ARTIFACTS)) { + const [platform, arch] = key.split('-'); + const artifact = artifactFor(platform, arch); + assert.equal(artifact.url, `https://liblibai-web-static.liblib.cloud/cli/${OFFICIAL_CLI_VERSION}/${expected.zipName}`); + assert.match(artifact.zipSHA256, /^[0-9a-f]{64}$/); + assert.match(artifact.binarySHA256, /^[0-9a-f]{64}$/); + assert.deepEqual( + [artifact.zipName, artifact.zipSHA256, artifact.binarySHA256], + AUDITED_ARTIFACTS[key], + ); + if (!key.startsWith('win32-')) assert.equal(artifact.executable, 'libtv'); + } + assert.equal(artifactFor('win32', 'x64').executable, 'libtv.exe'); + assert.throws(() => artifactFor('freebsd', 'x64'), /does not support/); +} + +async function testVerifiedInstallAndCacheReuse() { + const root = await mkdtemp(join(tmpdir(), 'pippit-libtv-bootstrap-test-')); + const zipBytes = Buffer.from('fake pinned zip'); + const binaryBytes = Buffer.from('#!/bin/sh\necho 1.1.3\n'); + const artifacts = fakeArtifact(zipBytes, binaryBytes); + let downloads = 0; + const download = async ({ destination }) => { + downloads += 1; + await writeFile(destination, zipBytes, { mode: 0o600 }); + }; + const extract = async ({ destination }) => { + const nested = join(destination, 'libtv-linux-x64'); + await mkdir(nested, { recursive: true }); + await writeFile(join(nested, 'libtv'), binaryBytes, { mode: 0o600 }); + }; + try { + const binary = await bootstrapLibTVCLI({ + platform: 'linux', + arch: 'x64', + artifacts, + cacheRoot: root, + download, + extract, + }); + assert.equal(await readFile(binary, 'utf8'), binaryBytes.toString()); + assert.equal((await stat(binary)).mode & 0o777, 0o700); + const metadataPath = join(dirname(binary), 'metadata.json'); + assert.equal((await stat(metadataPath)).mode & 0o777, 0o600); + assert.equal(JSON.parse(await readFile(metadataPath, 'utf8')).binary_sha256, sha256(binaryBytes)); + + const cached = await bootstrapLibTVCLI({ + platform: 'linux', + arch: 'x64', + artifacts, + cacheRoot: root, + download: async () => { throw new Error('cache was not reused'); }, + extract, + }); + assert.equal(cached, binary); + assert.equal(downloads, 1); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +async function testIntegrityFailures() { + const root = await mkdtemp(join(tmpdir(), 'pippit-libtv-bootstrap-failure-')); + const zipBytes = Buffer.from('zip'); + const binaryBytes = Buffer.from('binary'); + try { + await assert.rejects(bootstrapLibTVCLI({ + platform: 'linux', arch: 'x64', artifacts: fakeArtifact(zipBytes, binaryBytes), cacheRoot: '/', + }), /broad directory/); + + const badZipArtifacts = fakeArtifact(Buffer.from('different'), binaryBytes); + await assert.rejects(bootstrapLibTVCLI({ + platform: 'linux', arch: 'x64', artifacts: badZipArtifacts, cacheRoot: join(root, 'bad-zip'), + download: async ({ destination }) => writeFile(destination, zipBytes), + extract: async () => { throw new Error('must not extract a bad ZIP'); }, + }), /ZIP failed SHA-256/); + + const artifacts = fakeArtifact(zipBytes, binaryBytes); + await assert.rejects(bootstrapLibTVCLI({ + platform: 'linux', arch: 'x64', artifacts, cacheRoot: join(root, 'bad-binary'), + download: async ({ destination }) => writeFile(destination, zipBytes), + extract: async ({ destination }) => { + await mkdir(join(destination, 'bundle'), { recursive: true }); + await writeFile(join(destination, 'bundle', 'libtv'), 'tampered'); + }, + }), /binary failed SHA-256/); + + const cacheRoot = join(root, 'corrupt-cache'); + const binary = await bootstrapLibTVCLI({ + platform: 'linux', arch: 'x64', artifacts, cacheRoot, + download: async ({ destination }) => writeFile(destination, zipBytes), + extract: async ({ destination }) => { + await mkdir(join(destination, 'bundle'), { recursive: true }); + await writeFile(join(destination, 'bundle', 'libtv'), binaryBytes); + }, + }); + await writeFile(binary, 'tampered cache'); + await chmod(binary, 0o700); + await assert.rejects(bootstrapLibTVCLI({ + platform: 'linux', arch: 'x64', artifacts, cacheRoot, + download: async () => { throw new Error('must not replace corrupt cache'); }, + }), /cached LibTV CLI failed SHA-256/); + } finally { + await rm(root, { recursive: true, force: true }); + } +} + +await testOfficialArtifactMatrix(); +await testVerifiedInstallAndCacheReuse(); +await testIntegrityFailures(); +process.stdout.write('libtv bootstrap tests passed\n'); From e78c1596ac3205e78ea8b05f66e7ed5237ff74e9 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:48:30 +0800 Subject: [PATCH 18/48] feat: export LibTV URLs through official CLI Co-authored-by: Codex <codex@openai.com> --- adapters/libtv/cli.mjs | 39 +- adapters/libtv/exporter.mjs | 469 +++++++++++++++++++++ adapters/libtv/exporter.test.mjs | 278 ++++++++++++ adapters/libtv/testdata/fake-libtv-cli.mjs | 85 ++++ 4 files changed, 865 insertions(+), 6 deletions(-) create mode 100644 adapters/libtv/exporter.mjs create mode 100644 adapters/libtv/exporter.test.mjs create mode 100755 adapters/libtv/testdata/fake-libtv-cli.mjs diff --git a/adapters/libtv/cli.mjs b/adapters/libtv/cli.mjs index 44c7997..d658850 100755 --- a/adapters/libtv/cli.mjs +++ b/adapters/libtv/cli.mjs @@ -4,13 +4,20 @@ import { chmod, readFile, writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; +import { exportLibTVURL } from './exporter.mjs'; import { convertSnapshotToCanvasPlan } from './plan.mjs'; +const BOOLEAN_FLAGS = new Set(['non-interactive']); + function parseArgs(argv) { const [command, ...rest] = argv; const args = { command }; for (let index = 0; index < rest.length; index += 1) { const key = rest[index]; + if (key?.startsWith('--') && BOOLEAN_FLAGS.has(key.slice(2))) { + args[key.slice(2)] = true; + continue; + } const value = rest[index + 1]; if (!key?.startsWith('--') || !value || value.startsWith('--')) { throw new Error(`invalid argument near ${key ?? '<end>'}`); @@ -21,6 +28,18 @@ function parseArgs(argv) { return args; } +async function runExport(args) { + const result = await exportLibTVURL({ + url: required(args, 'url'), + outputDir: required(args, 'output-dir'), + binary: args['libtv-cli'], + nonInteractive: Boolean(args['non-interactive']), + title: args.title, + env: process.env, + }); + process.stdout.write(`${JSON.stringify(result)}\n`); +} + function required(args, key) { const value = args[key]?.trim(); if (!value) throw new Error(`--${key} is required`); @@ -63,13 +82,21 @@ async function runPlan(args) { async function main(argv = process.argv.slice(2)) { const args = parseArgs(argv); - if (args.command !== 'plan') { - throw new Error( - 'usage: node adapters/libtv/cli.mjs plan --snapshot <snapshot.json> ' + - '[--media-manifest <manifest.json>] [--title <title>] --output <plan.json|->', - ); + if (args.command === 'plan') { + await runPlan(args); + return; + } + if (args.command === 'export') { + await runExport(args); + return; } - await runPlan(args); + throw new Error( + 'usage:\n' + + ' node adapters/libtv/cli.mjs export --url <LibTV canvas URL> --output-dir <new directory> ' + + '[--libtv-cli <path>] [--non-interactive] [--title <title>]\n' + + ' node adapters/libtv/cli.mjs plan --snapshot <snapshot.json> ' + + '[--media-manifest <manifest.json>] [--title <title>] --output <plan.json|->', + ); } export { main, parseArgs }; diff --git a/adapters/libtv/exporter.mjs b/adapters/libtv/exporter.mjs new file mode 100644 index 0000000..79761d0 --- /dev/null +++ b/adapters/libtv/exporter.mjs @@ -0,0 +1,469 @@ +import { createHash } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { createReadStream } from 'node:fs'; +import { + chmod, + lstat, + mkdir, + mkdtemp, + readdir, + rename, + rm, + stat, + writeFile, +} from 'node:fs/promises'; +import { basename, dirname, extname, join, resolve } from 'node:path'; + +import { + OFFICIAL_CLI_VERSION, + OFFICIAL_INSTALLERS, + bootstrapLibTVCLI, +} from './bootstrap.mjs'; +import { convertSnapshotToCanvasPlan } from './plan.mjs'; + +const EXPORT_RESULT_SCHEMA = 'pippit-libtv-export-result/0.1'; +const MEDIA_MANIFEST_SCHEMA = 'pippit-libtv-media-manifest/0.1'; +const SUPPORTED_NODE_TYPES = new Set(['group', 'image', 'video', 'audio', 'video-clip']); +const MEDIA_NODE_TYPES = new Set(['image', 'video', 'audio']); +const MAX_CAPTURE_BYTES = 32 << 20; +const MEDIA_DOWNLOAD_ATTEMPTS = 3; +const CHILD_ENV_ALLOWLIST = new Set([ + 'HOME', 'USERPROFILE', 'APPDATA', 'LOCALAPPDATA', 'XDG_CONFIG_HOME', 'XDG_CACHE_HOME', + 'PATH', 'PATHEXT', 'SYSTEMROOT', 'COMSPEC', 'TMP', 'TEMP', 'TMPDIR', 'LANG', 'TZ', + 'DISPLAY', 'WAYLAND_DISPLAY', 'XDG_RUNTIME_DIR', 'DBUS_SESSION_BUS_ADDRESS', + 'SSL_CERT_FILE', 'SSL_CERT_DIR', 'LIBTV_CONFIG_DIR', +]); +const PROXY_ENV_KEYS = new Set(['HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY']); +const PROXY_PROTOCOLS = new Set(['http:', 'https:', 'socks:', 'socks4:', 'socks4a:', 'socks5:', 'socks5h:']); + +class LibTVExportError extends Error { + constructor(code, message) { + super(message); + this.name = 'LibTVExportError'; + this.code = code; + } +} + +function parseLibTVCanvasURL(value) { + let url; + try { + url = new URL(String(value)); + } catch { + throw new LibTVExportError('INVALID_URL', 'LibTV URL is invalid'); + } + if (url.protocol !== 'https:' || !['www.liblib.tv', 'liblib.tv'].includes(url.hostname) || url.pathname !== '/canvas') { + throw new LibTVExportError('INVALID_URL', 'expected an HTTPS LibTV canvas URL'); + } + if (url.username || url.password || url.searchParams.getAll('projectId').length !== 1) { + throw new LibTVExportError('INVALID_URL', 'LibTV URL must contain exactly one projectId and no credentials'); + } + const projectId = url.searchParams.get('projectId')?.trim() ?? ''; + if (!/^(?:[0-9a-f]{32}|[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})$/i.test(projectId)) { + throw new LibTVExportError('INVALID_URL', 'LibTV projectId must be a 32-hex ID or UUID'); + } + return { projectId }; +} + +function sanitizedChildEnvironment(input = process.env) { + const output = {}; + for (const [key, value] of Object.entries(input)) { + if (value === undefined) continue; + const upper = key.toUpperCase(); + if (CHILD_ENV_ALLOWLIST.has(upper) || upper.startsWith('LC_')) { + output[key] = value; + continue; + } + if (PROXY_ENV_KEYS.has(upper) && isSafeProxyURL(value)) output[key] = value; + } + return output; +} + +function isSafeProxyURL(value) { + try { + const url = new URL(String(value)); + return PROXY_PROTOCOLS.has(url.protocol) && Boolean(url.hostname) && + !url.username && !url.password && !url.search && !url.hash; + } catch { + return false; + } +} + +function createCommandRunner(binary, environment) { + return { + binary, + async capture(args) { + return runChild(binary, args, environment, false); + }, + async interactive(args) { + return runChild(binary, args, environment, true); + }, + }; +} + +function supportsExporterVersion(version) { + const current = String(version).split(/[+-]/, 1)[0].split('.').map(Number); + const minimum = OFFICIAL_CLI_VERSION.split('.').map(Number); + if (current.length !== 3 || current.some((value) => !Number.isInteger(value)) || current[0] !== minimum[0]) return false; + for (let index = 0; index < 3; index += 1) { + if (current[index] !== minimum[index]) return current[index] > minimum[index]; + } + return true; +} + +function runChild(binary, args, environment, interactive) { + return new Promise((resolveResult) => { + let stdout = ''; + let stderr = ''; + let overflow = false; + let settled = false; + const child = spawn(binary, args, { + env: environment, + shell: false, + stdio: interactive ? ['inherit', process.stderr, process.stderr] : ['ignore', 'pipe', 'pipe'], + }); + const finish = (result) => { + if (settled) return; + settled = true; + resolveResult({ stdout, stderr, ...result }); + }; + child.on('error', (error) => finish({ exitCode: null, signal: null, error })); + if (!interactive) { + const collect = (target) => (chunk) => { + const text = chunk.toString('utf8'); + if (stdout.length + stderr.length + text.length > MAX_CAPTURE_BYTES) { + overflow = true; + child.kill('SIGTERM'); + return; + } + if (target === 'stdout') stdout += text; + else stderr += text; + }; + child.stdout.on('data', collect('stdout')); + child.stderr.on('data', collect('stderr')); + } + child.on('close', (exitCode, signal) => finish({ exitCode, signal, overflow })); + }); +} + +async function locateLibTVCLI(options = {}) { + const environment = sanitizedChildEnvironment(options.env); + const override = options.binary ?? options.env?.LIBTV_CLI_BINARY ?? options.env?.LIBTV_CLI_PATH; + if (override) { + const runner = createCommandRunner(override, environment); + const result = await runner.capture(['--version']); + const version = result.exitCode === 0 ? result.stdout.trim().match(/\d+\.\d+\.\d+(?:[-+][\w.-]+)?/)?.[0] : undefined; + if (version && supportsExporterVersion(version)) return { runner, version }; + throw new LibTVExportError('CLI_UNAVAILABLE', `configured LibTV CLI is unavailable or invalid: ${override}`); + } + let bootstrapped; + try { + bootstrapped = await (options.bootstrap ?? bootstrapLibTVCLI)({ + env: options.env, + environment, + cacheRoot: options.cacheRoot, + }); + } catch (error) { + throw new LibTVExportError( + 'CLI_BOOTSTRAP_FAILED', + `LibTV CLI bootstrap failed: ${error instanceof Error ? error.message : String(error)}. ` + + `Official installer metadata: ${OFFICIAL_INSTALLERS.shell}`, + ); + } + const runner = createCommandRunner(bootstrapped, environment); + const result = await runner.capture(['--version']); + const version = result.exitCode === 0 ? result.stdout.trim().match(/\d+\.\d+\.\d+(?:[-+][\w.-]+)?/)?.[0] : undefined; + if (version !== OFFICIAL_CLI_VERSION) { + throw new LibTVExportError('CLI_BOOTSTRAP_FAILED', 'verified LibTV CLI cache returned an unexpected version'); + } + return { runner, version }; +} + +async function ensureAuthenticated(runner, nonInteractive) { + const probe = await runner.capture(['account', 'info']); + if (probe.exitCode === 0) return; + if (nonInteractive) { + throw new LibTVExportError('AUTH_REQUIRED', 'LibTV authentication is required; run `libtv login web --open` first'); + } + const login = await runner.interactive(['login', 'web', '--open']); + if (login.exitCode === 130 || login.signal) { + throw new LibTVExportError('LOGIN_CANCELLED', 'LibTV browser login was cancelled'); + } + if (login.exitCode !== 0) { + throw new LibTVExportError('LOGIN_FAILED', 'LibTV browser login did not complete'); + } + const verified = await runner.capture(['account', 'info']); + if (verified.exitCode !== 0) { + throw new LibTVExportError('LOGIN_FAILED', 'LibTV credentials were not available after browser login'); + } +} + +function parseCommandJSON(result, commandName) { + if (result.exitCode !== 0) { + throw new LibTVExportError('COMMAND_FAILED', `${commandName} failed (exit ${result.exitCode ?? 'spawn'})`); + } + if (result.overflow) throw new LibTVExportError('COMMAND_OUTPUT_TOO_LARGE', `${commandName} output exceeded the safety limit`); + try { + return JSON.parse(result.stdout); + } catch { + throw new LibTVExportError('INVALID_CLI_JSON', `${commandName} did not return valid JSON`); + } +} + +function validateProject(project, projectId) { + if (project?.projectUuid !== projectId || !Array.isArray(project?.nodes) || !Array.isArray(project?.edges)) { + throw new LibTVExportError('INVALID_PROJECT', 'LibTV project summary is incomplete or does not match the URL'); + } + const seen = new Set(); + for (const [index, node] of project.nodes.entries()) { + if (typeof node?.id !== 'string' || !node.id.trim() || seen.has(node.id)) { + throw new LibTVExportError('INVALID_PROJECT', `LibTV project node ${index} has a missing or duplicate ID`); + } + if (!SUPPORTED_NODE_TYPES.has(node.type)) { + throw new LibTVExportError('UNSUPPORTED_NODE', `unsupported LibTV node type: ${node.type ?? '<missing>'}`); + } + seen.add(node.id); + } +} + +function hasMediaResult(detail) { + const value = detail?.data?.url; + if (Array.isArray(value)) return value.some((item) => typeof item === 'string' && item.trim()); + return typeof value === 'string' && Boolean(value.trim()); +} + +function sanitizedExternalValue(value, key = '') { + if (/url|uri|token|cookie|authorization|credential|signature|secret|access.?key/i.test(key)) return undefined; + if (typeof value === 'string') { + if (/\b(?:https?|data|blob):/i.test(value)) return undefined; + return value; + } + if (Array.isArray(value)) { + return value.map((item) => sanitizedExternalValue(item)).filter((item) => item !== undefined); + } + if (!value || typeof value !== 'object') return value; + return Object.fromEntries( + Object.entries(value) + .map(([childKey, child]) => [childKey, sanitizedExternalValue(child, childKey)]) + .filter(([, child]) => child !== undefined), + ); +} + +function safeFileName(value, fallback) { + const normalized = String(value || '').replaceAll('\\', '/'); + const raw = basename(normalized).normalize('NFC').replaceAll(/[\u0000-\u001f\u007f/:*?"<>|]/g, '_'); + const characters = Array.from(raw); + const safe = characters.slice(0, 160).join('').replace(/^\.+$/, ''); + return safe || fallback; +} + +async function regularFilesUnder(root, current = root) { + const files = []; + for (const entry of await readdir(current, { withFileTypes: true })) { + const path = join(current, entry.name); + if (entry.isSymbolicLink()) throw new LibTVExportError('UNSAFE_DOWNLOAD', 'LibTV download produced a symbolic link'); + if (entry.isDirectory()) files.push(...await regularFilesUnder(root, path)); + else if (entry.isFile()) files.push(path); + else throw new LibTVExportError('UNSAFE_DOWNLOAD', 'LibTV download produced an unsupported filesystem entry'); + } + return files; +} + +async function fileSHA256(path) { + const hash = createHash('sha256'); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest('hex'); +} + +async function writePrivateJSON(path, value) { + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); + await chmod(path, 0o600); +} + +async function pathExists(path) { + try { + await lstat(path); + return true; + } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + +async function exportMedia(runner, tasks, projectId, stagingPath) { + const mediaDirectory = join(stagingPath, 'media'); + const downloadsDirectory = join(stagingPath, '.downloads'); + await mkdir(mediaDirectory, { mode: 0o700 }); + await mkdir(downloadsDirectory, { mode: 0o700 }); + const manifest = []; + const deduplicated = new Map(); + for (const [index, task] of tasks.entries()) { + let downloaded; + let lastFailure = 'command failed'; + for (let attempt = 0; attempt < MEDIA_DOWNLOAD_ATTEMPTS; attempt += 1) { + const downloadDirectory = join( + downloadsDirectory, + `${String(index).padStart(4, '0')}-attempt-${attempt + 1}`, + ); + await mkdir(downloadDirectory, { mode: 0o700 }); + const result = await runner.capture(['download', '-n', task.node.id, '-p', projectId, '-o', downloadDirectory]); + if (result.exitCode === 0) { + const files = await regularFilesUnder(downloadDirectory); + if (files.length === 1 && extname(files[0]).toLowerCase() !== '.zip') { + const fileInfo = await stat(files[0]); + if (fileInfo.isFile() && fileInfo.size > 0 && Number.isSafeInteger(fileInfo.size)) { + downloaded = { path: files[0], fileInfo }; + break; + } + lastFailure = 'produced an invalid media file'; + } else { + lastFailure = 'did not produce one direct media file'; + } + } else { + lastFailure = `failed with exit ${result.exitCode ?? 'spawn'}`; + } + if (attempt + 1 < MEDIA_DOWNLOAD_ATTEMPTS) { + await new Promise((resolveDelay) => setTimeout(resolveDelay, 200 * (2 ** attempt))); + } + } + if (!downloaded) { + throw new LibTVExportError( + 'MEDIA_DOWNLOAD_FAILED', + `LibTV ${task.node.type} download failed for node ${task.node.id} after ${MEDIA_DOWNLOAD_ATTEMPTS} attempts (${lastFailure})`, + ); + } + const digest = await fileSHA256(downloaded.path); + let stored = deduplicated.get(digest); + if (!stored) { + const originalName = safeFileName(downloaded.path, `${task.node.type}.bin`); + const nodePrefix = createHash('sha256').update(task.node.id).digest('hex').slice(0, 12); + const relativePath = `media/${nodePrefix}-${originalName}`; + await rename(downloaded.path, join(stagingPath, relativePath)); + await chmod(join(stagingPath, relativePath), 0o600); + stored = { relativePath, fileName: originalName, byteSize: downloaded.fileInfo.size }; + deduplicated.set(digest, stored); + } + manifest.push({ + logical_id: `media:${task.node.id}`, + source_node_id: task.node.id, + file_name: stored.fileName, + media_type: task.node.type, + relative_path: stored.relativePath, + sha256: digest, + byte_size: stored.byteSize, + }); + } + await rm(downloadsDirectory, { recursive: true, force: true }); + return manifest; +} + +async function exportLibTVURL(options) { + const { projectId } = parseLibTVCanvasURL(options.url); + const outputPath = resolve(options.outputDir); + if (await pathExists(outputPath)) { + throw new LibTVExportError('OUTPUT_EXISTS', `output directory already exists: ${outputPath}`); + } + const { runner, version } = await locateLibTVCLI({ + binary: options.binary, + env: options.env ?? process.env, + bootstrap: options.bootstrap, + cacheRoot: options.cacheRoot, + }); + await ensureAuthenticated(runner, Boolean(options.nonInteractive)); + const projectResult = await runner.capture(['project', projectId]); + if (projectResult.exitCode !== 0) { + throw new LibTVExportError('PROJECT_FORBIDDEN', 'LibTV project is unavailable or permission was denied'); + } + const project = parseCommandJSON(projectResult, 'libtv project'); + validateProject(project, projectId); + + const nodeDetails = []; + const mediaTasks = []; + const emptyMedia = []; + for (const node of project.nodes) { + const detailCommand = node.type === 'group' ? 'group' : 'node'; + const detail = parseCommandJSON( + await runner.capture([detailCommand, node.id, '-p', projectId]), + `libtv ${detailCommand} ${node.id}`, + ); + const downloadable = MEDIA_NODE_TYPES.has(node.type) && hasMediaResult(detail); + if (downloadable) mediaTasks.push({ node, detail }); + else if (node.type === 'image' || node.type === 'video') { + emptyMedia.push({ source_node_id: node.id, media_type: node.type, reason: 'source_has_no_media' }); + } else if (node.type === 'audio') { + throw new LibTVExportError('EMPTY_AUDIO', `LibTV audio node ${node.id} has no downloadable media`); + } + nodeDetails.push({ + sourceNodeId: node.id, + detail: sanitizedExternalValue(detail) ?? {}, + summary: { type: node.type, hasDownloadableMedia: downloadable }, + }); + } + + await mkdir(dirname(outputPath), { recursive: true, mode: 0o700 }); + const stagingPath = await mkdtemp(join(dirname(outputPath), `.${basename(outputPath)}.staging-`)); + await chmod(stagingPath, 0o700); + let completed = false; + try { + const media = await exportMedia(runner, mediaTasks, projectId, stagingPath); + const source = { platform: 'libtv', cliVersion: version, projectId }; + const snapshot = { + protocolVersion: 'xyq-libtv-snapshot/0.1', + exportedAt: new Date().toISOString(), + source, + project: sanitizedExternalValue(project), + nodeDetails, + assetReferences: [], + diagnostics: { nodeDetailsSucceeded: nodeDetails.length, nodeDetailsFailed: 0 }, + stats: { nodes: project.nodes.length, edges: project.edges.length, media: media.length, emptyMedia: emptyMedia.length }, + }; + const mediaManifest = { + schema: MEDIA_MANIFEST_SCHEMA, + source: { provider: 'libtv', project_id: projectId, cli_version: version }, + media, + empty_media: emptyMedia, + }; + const plan = convertSnapshotToCanvasPlan(snapshot, { mediaManifest, title: options.title }); + const serialized = JSON.stringify({ snapshot, mediaManifest, plan }); + if (/\b(?:https?|data|blob):/i.test(serialized)) { + throw new LibTVExportError('SANITIZATION_FAILED', 'sanitized LibTV bundle still contains an external URL'); + } + await writePrivateJSON(join(stagingPath, 'snapshot.json'), snapshot); + await writePrivateJSON(join(stagingPath, 'media-manifest.json'), mediaManifest); + await writePrivateJSON(join(stagingPath, 'plan.json'), plan); + await rename(stagingPath, outputPath); + completed = true; + return { + schema: EXPORT_RESULT_SCHEMA, + plan_schema: plan.schema, + bundle_dir: outputPath, + snapshot_path: join(outputPath, 'snapshot.json'), + media_manifest_path: join(outputPath, 'media-manifest.json'), + plan_path: join(outputPath, 'plan.json'), + source: plan.source, + media: media.map((item) => ({ + logical_id: item.logical_id, + media_type: item.media_type, + local_path: join(outputPath, item.relative_path), + })), + media_count: media.length, + node_count: plan.nodes.length, + group_count: plan.groups.length, + edge_count: plan.edges.length, + degradation_count: plan.degradations.length, + }; + } finally { + if (!completed) await rm(stagingPath, { recursive: true, force: true }); + } +} + +export { + EXPORT_RESULT_SCHEMA, + MEDIA_MANIFEST_SCHEMA, + LibTVExportError, + OFFICIAL_CLI_VERSION, + OFFICIAL_INSTALLERS, + exportLibTVURL, + locateLibTVCLI, + parseLibTVCanvasURL, + sanitizedChildEnvironment, +}; diff --git a/adapters/libtv/exporter.test.mjs b/adapters/libtv/exporter.test.mjs new file mode 100644 index 0000000..80e4afa --- /dev/null +++ b/adapters/libtv/exporter.test.mjs @@ -0,0 +1,278 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { chmod, lstat, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + EXPORT_RESULT_SCHEMA, + MEDIA_MANIFEST_SCHEMA, + exportLibTVURL, + locateLibTVCLI, + parseLibTVCanvasURL, + sanitizedChildEnvironment, +} from './exporter.mjs'; + +const fakeCLI = fileURLToPath(new URL('./testdata/fake-libtv-cli.mjs', import.meta.url)); +const adapterCLI = fileURLToPath(new URL('./cli.mjs', import.meta.url)); +const projectId = '0123456789abcdef0123456789abcdef'; +const projectURL = `https://www.liblib.tv/canvas?spaceId=123&projectId=${projectId}`; + +async function exists(path) { + try { + await lstat(path); + return true; + } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + +async function fixture(scenario) { + const root = await mkdtemp(join(tmpdir(), 'pippit-libtv-export-test-')); + const outputDir = join(root, 'bundle'); + const configDir = join(root, 'libtv-config'); + const logPath = join(root, 'commands.ndjson'); + const statePath = join(root, 'login-state'); + await mkdir(configDir, { mode: 0o700 }); + const configure = async (overrides = {}) => writeFile( + join(configDir, 'fake-cli-test.json'), + `${JSON.stringify({ scenario, logPath, statePath, ...overrides })}\n`, + { mode: 0o600 }, + ); + await configure(); + return { + root, + outputDir, + logPath, + statePath, + configure, + options: { + url: projectURL, + outputDir, + binary: fakeCLI, + env: { + ...process.env, + LIBTV_CONFIG_DIR: configDir, + XYQ_ACCESS_KEY: 'must-never-reach-libtv', + PIPPIT_ACCESS_KEY: 'must-never-reach-libtv', + PIPPIT_AK: 'must-never-reach-libtv', + THIRD_PARTY_API_KEY: 'must-never-reach-libtv', + UNKNOWN_TOKEN: 'must-never-reach-libtv', + SSH_AUTH_SOCK: join(root, 'must-never-reach-libtv.sock'), + }, + }, + }; +} + +async function readJson(path) { + return JSON.parse(await readFile(path, 'utf8')); +} + +function testChildEnvironmentAllowlist() { + const sanitized = sanitizedChildEnvironment({ + HOME: '/safe/home', + Path: '/safe/bin', + LC_ALL: 'C.UTF-8', + LIBTV_CONFIG_DIR: '/safe/libtv-config', + HTTP_PROXY: 'http://proxy.example:8080', + ALL_PROXY: 'socks5://proxy.example:1080', + HTTPS_PROXY: 'https://user:password@proxy.example:443', + FTP_PROXY: 'ftp://proxy.example', + THIRD_PARTY_API_KEY: 'secret', + UNKNOWN_TOKEN: 'secret', + MY_PASSWORD: 'secret', + SSH_AUTH_SOCK: '/tmp/agent.sock', + LIBTV_TOKEN: 'secret', + }); + assert.deepEqual(sanitized, { + HOME: '/safe/home', + Path: '/safe/bin', + LC_ALL: 'C.UTF-8', + LIBTV_CONFIG_DIR: '/safe/libtv-config', + HTTP_PROXY: 'http://proxy.example:8080', + ALL_PROXY: 'socks5://proxy.example:1080', + }); +} + +async function testAuthenticatedExport() { + const context = await fixture('authenticated'); + try { + const result = await exportLibTVURL(context.options); + assert.equal(result.schema, EXPORT_RESULT_SCHEMA); + assert.equal(result.plan_schema, 'pippit-canvas-plan/0.1'); + assert.equal(result.media_count, 3); + assert.equal(result.node_count, 4); + assert.equal(result.group_count, 1); + assert.equal(result.edge_count, 1); + assert.equal(result.degradation_count, 1); + assert.equal(result.media.length, 3); + for (const item of result.media) { + assert.equal(item.local_path.startsWith(`${context.outputDir}/media/`), true); + assert.equal(await exists(item.local_path), true); + } + + const snapshot = await readJson(result.snapshot_path); + const manifest = await readJson(result.media_manifest_path); + const plan = await readJson(result.plan_path); + assert.equal(manifest.schema, MEDIA_MANIFEST_SCHEMA); + assert.equal(manifest.media.length, 3); + assert.deepEqual(manifest.empty_media, [{ + source_node_id: 'video-empty', + media_type: 'video', + reason: 'source_has_no_media', + }]); + for (const item of manifest.media) { + assert.match(item.sha256, /^[0-9a-f]{64}$/); + assert.match(item.relative_path, /^media\/[a-z0-9-]+\.[a-z0-9]+$/); + assert.equal(item.byte_size > 0, true); + } + assert.equal(plan.required_media.length, 3); + for (const item of plan.required_media) { + assert.match(item.local_path, /^media\//); + assert.match(item.sha256, /^[0-9a-f]{64}$/); + assert.equal('url' in item, false); + } + assert.equal(plan.nodes.find((node) => node.source_node_id === 'image-1').kind, 'image'); + assert.equal(plan.nodes.find((node) => node.source_node_id === 'video-empty').kind, 'video-placeholder'); + assert.equal(plan.degradations[0].code, 'libtv.media.empty_placeholder'); + assert.equal(snapshot.assetReferences.length, 0); + + const serialized = [result.snapshot_path, result.media_manifest_path, result.plan_path] + .map((path) => readFile(path, 'utf8')); + const contents = (await Promise.all(serialized)).join('\n'); + for (const forbidden of ['https://', 'source-secret', 'must-never-reach-libtv', 'PIPPIT_ACCESS_KEY']) { + assert.equal(contents.includes(forbidden), false, `bundle leaked ${forbidden}`); + } + const commands = (await readFile(context.logPath, 'utf8')).trim().split('\n').map(JSON.parse); + assert.equal(commands.some((args) => args[0] === 'login'), false); + assert.equal(commands.some((args) => args[0] === 'group' && args[1] === 'group-1'), true); + } finally { + await rm(context.root, { recursive: true, force: true }); + } +} + +async function testBrowserLogin() { + const context = await fixture('login-required'); + try { + const result = await exportLibTVURL(context.options); + assert.equal(await exists(result.plan_path), true); + const commands = (await readFile(context.logPath, 'utf8')).trim().split('\n').map(JSON.parse); + assert.equal(commands.some((args) => args.join(' ') === 'login web --open'), true); + } finally { + await rm(context.root, { recursive: true, force: true }); + } +} + +async function testTransientMediaRetry() { + const context = await fixture('transient-media'); + try { + const result = await exportLibTVURL(context.options); + assert.equal(result.media_count, 3); + const commands = (await readFile(context.logPath, 'utf8')).trim().split('\n').map(JSON.parse); + assert.equal(commands.filter((args) => args[0] === 'download' && args[2] === 'image-1').length, 2); + } finally { + await rm(context.root, { recursive: true, force: true }); + } +} + +async function testLoginDoesNotPolluteJSONStdout() { + const context = await fixture('login-required'); + try { + await context.configure({ loginPrompt: true }); + const result = spawnSync(process.execPath, [ + adapterCLI, + 'export', + '--url', projectURL, + '--output-dir', context.outputDir, + '--libtv-cli', fakeCLI, + ], { encoding: 'utf8', env: context.options.env }); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).schema, EXPORT_RESULT_SCHEMA); + assert.equal(result.stdout.trim().split('\n').length, 1); + assert.match(result.stderr, /fake browser login prompt/); + } finally { + await rm(context.root, { recursive: true, force: true }); + } +} + +async function testLocateUsesVerifiedBootstrap() { + const context = await fixture('authenticated'); + try { + const pathHijackDirectory = join(context.root, 'path-hijack'); + const pathHijackMarker = join(pathHijackDirectory, 'executed'); + await mkdir(pathHijackDirectory, { mode: 0o700 }); + await writeFile( + join(pathHijackDirectory, 'libtv'), + `#!/bin/sh\nprintf executed > '${pathHijackMarker}'\nprintf '1.1.3\\n'\n`, + { mode: 0o700 }, + ); + let calls = 0; + const found = await locateLibTVCLI({ + env: { + ...context.options.env, + HOME: context.root, + PATH: `${pathHijackDirectory}:${dirname(process.execPath)}:/usr/bin:/bin`, + }, + bootstrap: async () => { + calls += 1; + return fakeCLI; + }, + }); + assert.equal(found.version, '1.1.3'); + assert.equal(found.runner.binary, fakeCLI); + assert.equal(calls, 1); + assert.equal(await exists(pathHijackMarker), false); + + const homeBinaryDirectory = join(context.root, '.libtv'); + const homeHijackMarker = join(homeBinaryDirectory, 'executed'); + await mkdir(homeBinaryDirectory, { mode: 0o700 }); + await writeFile( + join(homeBinaryDirectory, 'libtv'), + `#!/bin/sh\nprintf executed > '${homeHijackMarker}'\nprintf '1.1.3\\n'\n`, + { mode: 0o700 }, + ); + let homeCalls = 0; + await locateLibTVCLI({ + env: { ...context.options.env, HOME: context.root, PATH: `${dirname(process.execPath)}:/usr/bin:/bin` }, + bootstrap: async () => { + homeCalls += 1; + return fakeCLI; + }, + }); + assert.equal(homeCalls, 1); + assert.equal(await exists(homeHijackMarker), false); + } finally { + await rm(context.root, { recursive: true, force: true }); + } +} + +async function testClosedFailure(scenario, expected, extra = {}) { + const context = await fixture(scenario); + try { + await assert.rejects(exportLibTVURL({ ...context.options, ...extra }), expected); + assert.equal(await exists(context.outputDir), false); + } finally { + await rm(context.root, { recursive: true, force: true }); + } +} + +async function testFailureModes() { + await testClosedFailure('non-interactive', /authentication is required/, { nonInteractive: true }); + await testClosedFailure('login-cancel', /login was cancelled/); + await testClosedFailure('permission-denied', /permission was denied/); + await testClosedFailure('partial-media', /download failed/); + assert.throws(() => parseLibTVCanvasURL('https://evil.example/canvas?projectId=0123456789abcdef0123456789abcdef'), /expected an HTTPS LibTV canvas URL/); + assert.throws(() => parseLibTVCanvasURL('https://www.liblib.tv/canvas?projectId=bad'), /projectId/); +} + +await chmod(fakeCLI, 0o755); +testChildEnvironmentAllowlist(); +await testAuthenticatedExport(); +await testBrowserLogin(); +await testTransientMediaRetry(); +await testLoginDoesNotPolluteJSONStdout(); +await testLocateUsesVerifiedBootstrap(); +await testFailureModes(); +process.stdout.write('libtv URL exporter tests passed\n'); diff --git a/adapters/libtv/testdata/fake-libtv-cli.mjs b/adapters/libtv/testdata/fake-libtv-cli.mjs new file mode 100755 index 0000000..d8587c9 --- /dev/null +++ b/adapters/libtv/testdata/fake-libtv-cli.mjs @@ -0,0 +1,85 @@ +#!/usr/bin/env node + +import { appendFile, mkdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +const args = process.argv.slice(2); +let testConfig = {}; +if (process.env.LIBTV_CONFIG_DIR) { + try { + testConfig = JSON.parse(await readFile(join(process.env.LIBTV_CONFIG_DIR, 'fake-cli-test.json'), 'utf8')); + } catch { + // A missing test config represents an authenticated default fake. + } +} +const scenario = testConfig.scenario ?? 'authenticated'; +const statePath = testConfig.statePath; +const logPath = testConfig.logPath; + +if (process.env.THIRD_PARTY_API_KEY || process.env.UNKNOWN_TOKEN || process.env.SSH_AUTH_SOCK) process.exit(91); +if (logPath) await appendFile(logPath, `${JSON.stringify(args)}\n`); + +async function stateExists() { + if (!statePath) return false; + try { + await stat(statePath); + return true; + } catch { + return false; + } +} + +function output(value) { + process.stdout.write(`${typeof value === 'string' ? value : JSON.stringify(value)}\n`); +} + +if (args[0] === '--version') { + output('1.1.3'); +} else if (args[0] === 'account' && args[1] === 'info') { + const needsLogin = scenario === 'login-required' || scenario === 'login-cancel' || scenario === 'non-interactive'; + if (needsLogin && !(await stateExists())) process.exit(2); + output({ user: { id: 'fake' }, activeAccount: { accountType: 'personal' }, teamId: null, accountsCount: 1 }); +} else if (args[0] === 'login' && args[1] === 'web') { + if (scenario === 'login-cancel') process.exit(130); + if (testConfig.loginPrompt) output('fake browser login prompt'); + if (statePath) await writeFile(statePath, 'authenticated\n'); +} else if (args[0] === 'project') { + if (scenario === 'permission-denied') process.exit(3); + const projectUuid = args[1]; + output({ + projectUuid, + nodes: [ + { id: 'group-1', name: 'Sources', type: 'group', position: { x: 0, y: 0 }, width: 800, height: 700 }, + { id: 'image-1', name: 'Cover', type: 'image', position: { x: 20, y: 20 }, width: 320, height: 320, parentId: 'group-1' }, + { id: 'video-empty', name: 'Pending shot', type: 'video', position: { x: 400, y: 20 }, width: 320, height: 320, parentId: 'group-1' }, + { id: 'video-1', name: 'Shot', type: 'video', position: { x: 900, y: 20 }, width: 622, height: 350 }, + { id: 'audio-1', name: 'Voice', type: 'audio', position: { x: 900, y: 420 }, width: 350, height: 148 }, + ], + edges: [{ id: 'edge-1', source: 'image-1', target: 'video-1' }], + }); +} else if (args[0] === 'node' || args[0] === 'group') { + const nodeId = args[1]; + const details = { + 'group-1': { nodeKey: nodeId, name: 'Sources', data: { type: 'group', childNodeIds: ['image-1', 'video-empty'] } }, + 'image-1': { nodeKey: nodeId, name: 'Cover', data: { type: 'image', url: ['https://signed.example.test/cover.png?token=source-secret'], resourceMeta: { items: [{ extension: 'png', mimeType: 'image/png', width: 320, height: 320 }] } } }, + 'video-empty': { nodeKey: nodeId, name: 'Pending shot', data: { type: 'video', url: [] } }, + 'video-1': { nodeKey: nodeId, name: 'Shot', data: { type: 'video', url: ['https://signed.example.test/shot.mp4?signature=source-secret'], poster: 'https://signed.example.test/poster.jpg?token=source-secret', resourceMeta: { items: [{ extension: 'mp4', mimeType: 'video/mp4', durationSec: 2 }] } } }, + 'audio-1': { nodeKey: nodeId, name: 'Voice', data: { type: 'audio', url: ['https://signed.example.test/voice.wav?token=source-secret'], resourceMeta: { items: [{ extension: 'wav', mimeType: 'audio/wav', durationSec: 2 }] } } }, + }; + if (!details[nodeId]) process.exit(4); + output(details[nodeId]); +} else if (args[0] === 'download') { + const nodeId = args[args.indexOf('-n') + 1]; + const outputDirectory = args[args.indexOf('-o') + 1]; + if (scenario === 'partial-media' && nodeId === 'audio-1') process.exit(5); + if (scenario === 'transient-media' && nodeId === 'image-1' && !(await stateExists())) { + if (statePath) await writeFile(statePath, 'first-download-failed\n'); + process.exit(5); + } + const extensions = { 'image-1': 'png', 'video-1': 'mp4', 'audio-1': 'wav' }; + if (!extensions[nodeId]) process.exit(6); + await mkdir(outputDirectory, { recursive: true }); + await writeFile(join(outputDirectory, `${nodeId}.${extensions[nodeId]}`), `fake-${nodeId}-media\n`); +} else { + process.exit(64); +} From e36a01a821b85dd28e2690e0e9490ae80add1f4b Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 16:48:49 +0800 Subject: [PATCH 19/48] docs: document verified LibTV imports Co-authored-by: Codex <codex@openai.com> --- README.md | 26 +++++++++++++-- adapters/libtv/README.md | 69 +++++++++++++++++++++++++++++++++------- package.json | 2 +- 3 files changed, 82 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 142667f..5059534 100644 --- a/README.md +++ b/README.md @@ -221,9 +221,29 @@ pippit-tool-cli canvas apply --project-id PROJECT_ID --file ./patch.json PPE 只影响 Pippit API 同源请求。CLI 不会把 Access Key、`x-tt-env`、`x-use-ppe` 或 `x-schedule-vdc` 转发给第三方绝对 URL；`--ppe-env` 的优先级高于 `PIPPIT_CLI_PPE_ENV`，二者都未提供时访问生产环境。 -### LibTV 本地 adapter +### 一键导入 LibTV 画布 -beta 同时提供一个无网络的 LibTV provider adapter，将导出的 snapshot 转成 ID-neutral `pippit-canvas-plan/0.1`： +LibTV 迁移只是上述通用画布能力的 CLI 编排层，服务端不识别 LibTV。给定一个 LibTV 画布链接后，CLI 会通过官方 LibTV CLI 完成网页授权与草稿/素材导出，再依次调用通用的 `upload`、`create`、内部 ID 分配、单 transaction `apply` 和 `get` 全量校验： + +```bash +pippit-tool-cli \ + --ppe-env ppe_cli_canvas_ak \ + canvas import \ + --from libtv \ + --url 'https://www.liblib.tv/canvas?projectId=<project-id>' \ + --accept-degradations \ + --open +``` + +生产环境使用时删除 `--ppe-env ppe_cli_canvas_ak`。当前登录能力仍沿用既有 Access Key 配置；CLI 自动保存 AK 的 `login` 流程会单独交付。 + +首次运行时，若本机没有 LibTV CLI，导入器只会从 LibTV 官方静态域下载固定版本 1.1.3 的对应平台 ZIP，并同时校验 ZIP 和可执行文件的内置 SHA-256；不会执行远程安装脚本。若官方 LibTV CLI 尚未登录，它会打开 `libtv login web --open` 的官方网页授权流程，导入器本身不读取浏览器 Cookie 或 LibTV credential 文件。 + +`--accept-degradations` 表示接受计划中明确列出的不可移植节点。例如没有生成结果的图片/视频节点会保留为空占位，LibTV 私有 `video-clip` 会降级成空的 Pippit video-composite。未显式接受时，CLI 会在任何 Pippit 写入前停止。 + +导入状态会写入权限为 `0600` 的本地 journal。素材上传、画布创建或 transaction 结果不明确时，CLI 会保留已获得的持久 ID 并拒绝盲目重复写入；重复执行同一条命令会优先 query-back 恢复。只有 root 和所有伴生资产逐一通过 canonical hash 校验后，命令才返回 `state=verified` 并执行 `--open`。 + +需要单独检查或生成 ID-neutral `pippit-canvas-plan/0.1` 时，仍可使用纯本地 adapter： ```bash pippit-tool-cli libtv plan \ @@ -233,7 +253,7 @@ pippit-tool-cli libtv plan \ --output ./canvas-plan.json ``` -adapter 不读取 Pippit AK、不访问 LibTV、不分配 Pippit ID，也不执行 create/apply。后续 executor 只需把 plan 编译为上述通用 Canvas 命令即可。若 snapshot 只提供带签名参数的素材 URL，plan 会以 `0600` 权限保留它们以供后续下载；不要把 plan 打进日志、提交到仓库或分享给他人。完整边界见 `adapters/libtv/README.md`。 +完整 provider 边界见 `adapters/libtv/README.md`。 ## 短剧工作流技能 diff --git a/adapters/libtv/README.md b/adapters/libtv/README.md index 8b6f323..be6c997 100644 --- a/adapters/libtv/README.md +++ b/adapters/libtv/README.md @@ -5,7 +5,52 @@ LibTV snapshot into the ID-neutral `pippit-canvas-plan/0.1` contract. It does not read an access key, choose a Pippit environment, allocate Pippit asset IDs, upload files, create a project, write assets, bind a canvas, or use team state. -Generate a plan: +Export a LibTV URL with the official LibTV CLI, then generate a plan: + +```bash +node adapters/libtv/cli.mjs export \ + --url 'https://www.liblib.tv/canvas?projectId=<project-id>' \ + --output-dir ./libtv-bundle +``` + +An explicit `--libtv-cli`, `LIBTV_CLI_BINARY`, or `LIBTV_CLI_PATH` opts into a +user-managed binary after a version check. Without an explicit override, the +exporter never executes `libtv` from `PATH` or `~/.libtv`; it goes directly to +the verified cache/bootstrap path. This prevents an ambient binary from +bypassing the pin. Bootstrap installs the pinned official 1.1.3 ZIP into the +private Pippit tool cache and supports +Darwin, Linux, and Windows on arm64/x64; it verifies both ZIP and binary against +embedded SHA-256 values, uses `0700` directories/binary, and never executes a +remote installer or script. Windows extraction uses the built-in `tar.exe` +(Windows 10+); absence of a safe local extractor fails closed. Set +`PIPPIT_CLI_LIBTV_CACHE_DIR` to override the cache root. + +After locating a verified CLI, the exporter probes existing official CLI +credentials with `libtv account info`. If none are available, interactive use +runs `libtv login web --open`; `--non-interactive` instead fails with an +actionable login message. Login child output is redirected to stderr so stdout +remains one machine-readable JSON object. The adapter never reads browser +cookies or the LibTV credential file. The LibTV child receives only a small +runtime/login environment allowlist. Unknown variables, SSH agent sockets, and +all unlisted key/token/password values are omitted. HTTP/HTTPS/SOCKS proxy URLs +are passed only when they contain no user information. + +The output directory must not already exist. Export is staged privately and +renamed atomically, so cancellation, permission denial, or a partial media +download leaves no final bundle. The successful stdout object uses +`pippit-libtv-export-result/0.1` and returns `plan_path`, `snapshot_path`, +`media_manifest_path`, and absolute local paths for each media item. + +The bundle contains: + +- a URL- and credential-sanitized `snapshot.json`; +- `media-manifest.json` (`pippit-libtv-media-manifest/0.1`) with bundle-relative + paths, byte sizes, and bare lowercase SHA-256 digests; +- local files downloaded through official `libtv download`, preserving LibTV's + source-account permission and watermark behavior; +- `plan.json` (`pippit-canvas-plan/0.1`). + +To convert an existing snapshot instead: ```bash node adapters/libtv/cli.mjs plan \ @@ -16,16 +61,15 @@ node adapters/libtv/cli.mjs plan \ ``` `--media-manifest` is optional. It may provide `sourceNodeId` + `fileName` rows -for a local export bundle. Existing prototype manifests may also contain -Pippit IDs or authorization metadata; the adapter deliberately ignores those -fields and never copies them into the plan. If no manifest row exists, the -adapter uses the snapshot's HTTPS media reference and derives a file name. +for an older export, or `source_node_id`, `relative_path`, `sha256`, and +`media_type` rows for a local bundle. Existing prototype manifests may also +contain Pippit IDs or authorization metadata; the adapter deliberately ignores +those fields and never copies them into the plan. -The generated plan is written with mode `0600`. When the source export only -contains signed HTTPS media URLs, those URLs (including their query strings) -must remain in the local plan so a later executor can download the files. Treat -the plan as sensitive local state: do not print it into logs, commit it, or -share it. Prefer a local export bundle plus `--media-manifest` when available. +The generated plan is written with mode `0600`. Official URL export always +uses bundle-relative `local_path` + `sha256` and omits source URLs. Legacy +snapshot-only conversion may still accept an absolute HTTPS media URL, but its +fingerprint strips query strings and authentication fields. The generic canvas executor owns the remaining steps: @@ -43,7 +87,10 @@ snapshot and media mapping produce byte-for-byte stable JSON; the executor should hash the complete plan when deriving its operation identity. The v0.1 adapter fails closed for unsupported node types or dangling edges. -Supported source types are `group`, `video`, `audio`, and `video-clip`. +Supported source types are `group`, `image`, `video`, `audio`, and +`video-clip`. Empty LibTV image/video generation nodes are preserved as +`image-placeholder` / `video-placeholder` with an explicit degradation; they +are not mistaken for partially downloaded media. LibTV `video-clip` nodes do not carry a portable generated result. The plan preserves their input references and records an explicit degradation to an diff --git a/package.json b/package.json index 4ed2119..7f8ba54 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ }, "scripts": { "postinstall": "node scripts/install.js", - "test": "node scripts/version-check.test.js && node scripts/skills.test.js && node scripts/install-wizard.test.js && node scripts/run.test.js && node adapters/libtv/plan.test.mjs && go test ./... && go vet ./..." + "test": "node scripts/version-check.test.js && node scripts/skills.test.js && node scripts/install-wizard.test.js && node scripts/run.test.js && node adapters/libtv/bootstrap.test.mjs && node adapters/libtv/plan.test.mjs && node adapters/libtv/exporter.test.mjs && go test ./... && go vet ./..." }, "os": [ "darwin", From 263c6485cb9ce5ca7c05696618fee4d2ce0d9aba Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 17:33:38 +0800 Subject: [PATCH 20/48] fix(canvas): add guided import progress Co-authored-by: Codex <codex@openai.com> --- README.md | 18 +- adapters/libtv/README.md | 5 +- adapters/libtv/cli.mjs | 1 + adapters/libtv/exporter.mjs | 43 ++++- adapters/libtv/exporter.test.mjs | 22 +++ cmd/canvas/import.go | 75 ++++++-- cmd/canvas/import_journal.go | 64 +++++++ .../import_journal_security_unix_test.go | 86 +++++++++ cmd/canvas/import_media.go | 47 ++++- cmd/canvas/import_prompt.go | 134 +++++++++++++ cmd/canvas/import_test.go | 181 ++++++++++++++++++ internal/canvasplan/journal_preflight.go | 43 +++++ 12 files changed, 685 insertions(+), 34 deletions(-) create mode 100644 cmd/canvas/import_journal.go create mode 100644 cmd/canvas/import_journal_security_unix_test.go create mode 100644 cmd/canvas/import_prompt.go create mode 100644 internal/canvasplan/journal_preflight.go diff --git a/README.md b/README.md index 5059534..4398979 100644 --- a/README.md +++ b/README.md @@ -223,23 +223,23 @@ PPE 只影响 Pippit API 同源请求。CLI 不会把 Access Key、`x-tt-env`、 ### 一键导入 LibTV 画布 -LibTV 迁移只是上述通用画布能力的 CLI 编排层，服务端不识别 LibTV。给定一个 LibTV 画布链接后，CLI 会通过官方 LibTV CLI 完成网页授权与草稿/素材导出，再依次调用通用的 `upload`、`create`、内部 ID 分配、单 transaction `apply` 和 `get` 全量校验： +LibTV 迁移只是上述通用画布能力的 CLI 编排层，服务端不识别 LibTV。普通用户只需启动交互式导入： ```bash -pippit-tool-cli \ - --ppe-env ppe_cli_canvas_ak \ - canvas import \ - --from libtv \ - --url 'https://www.liblib.tv/canvas?projectId=<project-id>' \ - --accept-degradations \ - --open +pippit-tool-cli --ppe-env ppe_cli_canvas_ak canvas import ``` +CLI 会逐步询问来源、LibTV 链接、journal 位置、降级接受与是否打开结果。journal 直接回车即使用权限受控的自动路径，不需要设置环境变量。源端节点处理、素材下载与 Pippit 素材上传会在 stderr 显示已处理/总数/剩余数，画布创建、写入和回读校验会显示当前阶段；最终 stdout 仍只输出一行 JSON。 + +供 Agent、CI 或其它非交互场景使用时，仍可显式传入 `--from`、`--url`、`--accept-degradations` 和 `--open`；`--journal` 始终可选，省略时使用自动路径。 + +给定链接后，CLI 会通过官方 LibTV CLI 完成网页授权与草稿/素材导出，再依次调用通用的 `upload`、`create`、内部 ID 分配、单 transaction `apply` 和 `get` 全量校验。 + 生产环境使用时删除 `--ppe-env ppe_cli_canvas_ak`。当前登录能力仍沿用既有 Access Key 配置；CLI 自动保存 AK 的 `login` 流程会单独交付。 首次运行时，若本机没有 LibTV CLI，导入器只会从 LibTV 官方静态域下载固定版本 1.1.3 的对应平台 ZIP，并同时校验 ZIP 和可执行文件的内置 SHA-256；不会执行远程安装脚本。若官方 LibTV CLI 尚未登录，它会打开 `libtv login web --open` 的官方网页授权流程，导入器本身不读取浏览器 Cookie 或 LibTV credential 文件。 -`--accept-degradations` 表示接受计划中明确列出的不可移植节点。例如没有生成结果的图片/视频节点会保留为空占位，LibTV 私有 `video-clip` 会降级成空的 Pippit video-composite。未显式接受时，CLI 会在任何 Pippit 写入前停止。 +`--accept-degradations` 表示接受计划中明确列出的不可移植节点。例如没有生成结果的图片/视频节点会保留为空占位，LibTV 私有 `video-clip` 会降级成空的 Pippit video-composite。交互式导入会就地询问是否接受；非交互调用未传该参数时，CLI 会在任何 Pippit 写入前停止。 导入状态会写入权限为 `0600` 的本地 journal。素材上传、画布创建或 transaction 结果不明确时，CLI 会保留已获得的持久 ID 并拒绝盲目重复写入；重复执行同一条命令会优先 query-back 恢复。只有 root 和所有伴生资产逐一通过 canonical hash 校验后，命令才返回 `state=verified` 并执行 `--open`。 diff --git a/adapters/libtv/README.md b/adapters/libtv/README.md index be6c997..0ce6c0e 100644 --- a/adapters/libtv/README.md +++ b/adapters/libtv/README.md @@ -29,7 +29,10 @@ After locating a verified CLI, the exporter probes existing official CLI credentials with `libtv account info`. If none are available, interactive use runs `libtv login web --open`; `--non-interactive` instead fails with an actionable login message. Login child output is redirected to stderr so stdout -remains one machine-readable JSON object. The adapter never reads browser +remains one machine-readable JSON object. Project, node-detail, media-download, +and final count progress also goes to stderr. Phase lines precede CLI setup, +authentication, and project fetch; every media download emits a start line +before the potentially long transfer. The adapter never reads browser cookies or the LibTV credential file. The LibTV child receives only a small runtime/login environment allowlist. Unknown variables, SSH agent sockets, and all unlisted key/token/password values are omitted. HTTP/HTTPS/SOCKS proxy URLs diff --git a/adapters/libtv/cli.mjs b/adapters/libtv/cli.mjs index d658850..e3b461c 100755 --- a/adapters/libtv/cli.mjs +++ b/adapters/libtv/cli.mjs @@ -36,6 +36,7 @@ async function runExport(args) { nonInteractive: Boolean(args['non-interactive']), title: args.title, env: process.env, + onProgress: (message) => process.stderr.write(`${message}\n`), }); process.stdout.write(`${JSON.stringify(result)}\n`); } diff --git a/adapters/libtv/exporter.mjs b/adapters/libtv/exporter.mjs index 79761d0..3974bc9 100644 --- a/adapters/libtv/exporter.mjs +++ b/adapters/libtv/exporter.mjs @@ -88,6 +88,15 @@ function isSafeProxyURL(value) { } } +function reportProgress(reporter, message) { + if (typeof reporter !== 'function') return; + try { + reporter(`[libtv] ${message}`); + } catch { + // Progress reporting must not change export semantics. + } +} + function createCommandRunner(binary, environment) { return { binary, @@ -289,14 +298,22 @@ async function pathExists(path) { } } -async function exportMedia(runner, tasks, projectId, stagingPath) { +async function exportMedia(runner, tasks, projectId, stagingPath, onProgress) { const mediaDirectory = join(stagingPath, 'media'); const downloadsDirectory = join(stagingPath, '.downloads'); await mkdir(mediaDirectory, { mode: 0o700 }); await mkdir(downloadsDirectory, { mode: 0o700 }); const manifest = []; const deduplicated = new Map(); + if (tasks.length === 0) { + reportProgress(onProgress, 'media downloads: processed=0/0, remaining=0'); + } for (const [index, task] of tasks.entries()) { + reportProgress( + onProgress, + `media download start: current=${index + 1}/${tasks.length}, processed=${index}, ` + + `remaining=${tasks.length - index - 1}`, + ); let downloaded; let lastFailure = 'command failed'; for (let attempt = 0; attempt < MEDIA_DOWNLOAD_ATTEMPTS; attempt += 1) { @@ -351,6 +368,10 @@ async function exportMedia(runner, tasks, projectId, stagingPath) { sha256: digest, byte_size: stored.byteSize, }); + reportProgress( + onProgress, + `media downloads: processed=${index + 1}/${tasks.length}, remaining=${tasks.length - index - 1}`, + ); } await rm(downloadsDirectory, { recursive: true, force: true }); return manifest; @@ -362,24 +383,31 @@ async function exportLibTVURL(options) { if (await pathExists(outputPath)) { throw new LibTVExportError('OUTPUT_EXISTS', `output directory already exists: ${outputPath}`); } + reportProgress(options.onProgress, 'phase: preparing verified LibTV CLI'); const { runner, version } = await locateLibTVCLI({ binary: options.binary, env: options.env ?? process.env, bootstrap: options.bootstrap, cacheRoot: options.cacheRoot, }); + reportProgress(options.onProgress, 'phase: checking LibTV authentication'); await ensureAuthenticated(runner, Boolean(options.nonInteractive)); + reportProgress(options.onProgress, 'phase: fetching LibTV project summary'); const projectResult = await runner.capture(['project', projectId]); if (projectResult.exitCode !== 0) { throw new LibTVExportError('PROJECT_FORBIDDEN', 'LibTV project is unavailable or permission was denied'); } const project = parseCommandJSON(projectResult, 'libtv project'); validateProject(project, projectId); + reportProgress( + options.onProgress, + `project summary: nodes=${project.nodes.length}, edges=${project.edges.length}`, + ); const nodeDetails = []; const mediaTasks = []; const emptyMedia = []; - for (const node of project.nodes) { + for (const [index, node] of project.nodes.entries()) { const detailCommand = node.type === 'group' ? 'group' : 'node'; const detail = parseCommandJSON( await runner.capture([detailCommand, node.id, '-p', projectId]), @@ -397,6 +425,10 @@ async function exportLibTVURL(options) { detail: sanitizedExternalValue(detail) ?? {}, summary: { type: node.type, hasDownloadableMedia: downloadable }, }); + reportProgress( + options.onProgress, + `node details: processed=${index + 1}/${project.nodes.length}, remaining=${project.nodes.length - index - 1}`, + ); } await mkdir(dirname(outputPath), { recursive: true, mode: 0o700 }); @@ -404,7 +436,7 @@ async function exportLibTVURL(options) { await chmod(stagingPath, 0o700); let completed = false; try { - const media = await exportMedia(runner, mediaTasks, projectId, stagingPath); + const media = await exportMedia(runner, mediaTasks, projectId, stagingPath, options.onProgress); const source = { platform: 'libtv', cliVersion: version, projectId }; const snapshot = { protocolVersion: 'xyq-libtv-snapshot/0.1', @@ -432,6 +464,11 @@ async function exportLibTVURL(options) { await writePrivateJSON(join(stagingPath, 'plan.json'), plan); await rename(stagingPath, outputPath); completed = true; + reportProgress( + options.onProgress, + `export complete: nodes=${plan.nodes.length}, groups=${plan.groups.length}, edges=${plan.edges.length}, ` + + `media=${media.length}, degradations=${plan.degradations.length}`, + ); return { schema: EXPORT_RESULT_SCHEMA, plan_schema: plan.schema, diff --git a/adapters/libtv/exporter.test.mjs b/adapters/libtv/exporter.test.mjs index 80e4afa..7a1cd20 100644 --- a/adapters/libtv/exporter.test.mjs +++ b/adapters/libtv/exporter.test.mjs @@ -191,7 +191,29 @@ async function testLoginDoesNotPolluteJSONStdout() { assert.equal(result.status, 0, result.stderr); assert.equal(JSON.parse(result.stdout).schema, EXPORT_RESULT_SCHEMA); assert.equal(result.stdout.trim().split('\n').length, 1); + assert.equal(result.stdout.includes('[libtv]'), false); assert.match(result.stderr, /fake browser login prompt/); + assert.deepEqual( + result.stderr.trim().split('\n').filter((line) => line.startsWith('[libtv]')), + [ + '[libtv] phase: preparing verified LibTV CLI', + '[libtv] phase: checking LibTV authentication', + '[libtv] phase: fetching LibTV project summary', + '[libtv] project summary: nodes=5, edges=1', + '[libtv] node details: processed=1/5, remaining=4', + '[libtv] node details: processed=2/5, remaining=3', + '[libtv] node details: processed=3/5, remaining=2', + '[libtv] node details: processed=4/5, remaining=1', + '[libtv] node details: processed=5/5, remaining=0', + '[libtv] media download start: current=1/3, processed=0, remaining=2', + '[libtv] media downloads: processed=1/3, remaining=2', + '[libtv] media download start: current=2/3, processed=1, remaining=1', + '[libtv] media downloads: processed=2/3, remaining=1', + '[libtv] media download start: current=3/3, processed=2, remaining=0', + '[libtv] media downloads: processed=3/3, remaining=0', + '[libtv] export complete: nodes=4, groups=1, edges=1, media=3, degradations=1', + ], + ); } finally { await rm(context.root, { recursive: true, force: true }); } diff --git a/cmd/canvas/import.go b/cmd/canvas/import.go index 54d7b0e..6bf64f4 100644 --- a/cmd/canvas/import.go +++ b/cmd/canvas/import.go @@ -20,11 +20,14 @@ import ( ) type importOptions struct { - Provider string - SourceURL string - Open bool - AcceptDegradations bool - JournalPath string + Provider string + SourceURL string + Open bool + AcceptDegradations bool + JournalPath string + OpenExplicit bool + AcceptDegradationsExplicit bool + JournalExplicit bool } var ( @@ -49,6 +52,7 @@ type importDependencies struct { userConfigDir func() (string, error) target func() string authScope func() string + isInteractive func(io.Reader) bool } type runnerImportExecutor struct { @@ -74,6 +78,7 @@ func newImportDependencies(runner *common.Runner) importDependencies { userConfigDir: os.UserConfigDir, target: func() string { return canvasImportTarget(runner) }, authScope: func() string { return canvasImportAuthScope(runner) }, + isInteractive: importInputIsInteractive, } } @@ -85,9 +90,21 @@ func newImportCommand( cmd := &cobra.Command{ Use: "import", Short: "Import an external project into a personal novel Canvas", - Args: cobra.NoArgs, + Long: "Import an external project into a personal novel Canvas. " + + "Run without source flags for a guided import; flags remain available for Agent and CI automation.", + Example: " pippit-tool-cli --ppe-env ppe_cli_canvas_ak canvas import", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - result, err := runCanvasImport(cmd.Context(), opts, dependencies, stderr) + opts.OpenExplicit = cmd.Flags().Changed("open") + opts.AcceptDegradationsExplicit = cmd.Flags().Changed("accept-degradations") + opts.JournalExplicit = cmd.Flags().Changed("journal") + prepared, prompts, err := prepareCanvasImportOptions( + cmd.InOrStdin(), opts, dependencies.isInteractive, stderr, + ) + if err != nil { + return err + } + result, err := runCanvasImport(cmd.Context(), prepared, dependencies, stderr, prompts) if result != nil { if writeErr := common.WriteJSON(stdout, result); writeErr != nil { return writeErr @@ -95,8 +112,8 @@ func newImportCommand( } if err != nil { logCanvasError("canvas import", err, map[string]string{ - "provider": strings.TrimSpace(opts.Provider), - "journal": filepath.Base(strings.TrimSpace(opts.JournalPath)), + "provider": strings.TrimSpace(prepared.Provider), + "journal": filepath.Base(strings.TrimSpace(prepared.JournalPath)), }) return err } @@ -119,6 +136,7 @@ func runCanvasImport( opts importOptions, dependencies importDependencies, stderr io.Writer, + prompts *importPromptSession, ) (*canvasplan.ExecutionResult, error) { if strings.ToLower(strings.TrimSpace(opts.Provider)) != "libtv" { return nil, fmt.Errorf("canvas import --from must be libtv") @@ -127,11 +145,18 @@ func runCanvasImport( if err != nil { return nil, err } + explicitJournal, err := preflightExplicitImportJournal(opts.JournalPath, opts.JournalExplicit) + if err != nil { + return nil, err + } + if explicitJournal != "" { + opts.JournalPath = explicitJournal + } bundleRoot, outputDir, err := newImportBundlePath(dependencies.userCacheDir) if err != nil { return nil, err } - fmt.Fprintln(stderr, "Exporting the LibTV canvas and its media...") + fmt.Fprintln(stderr, "Phase export: exporting the LibTV canvas and its media...") exported, err := dependencies.exporter.Export(ctx, sourceURL, outputDir, stderr) if err != nil { return nil, fmt.Errorf("export LibTV canvas: %w", err) @@ -150,10 +175,22 @@ func runCanvasImport( return nil, err } if len(plan.Degradations) > 0 && !opts.AcceptDegradations { - return nil, fmt.Errorf( - "LibTV export reports %d explicit degradation(s); inspect %s (plan: %s), then rerun with --accept-degradations", - len(plan.Degradations), outputDir, exported.PlanPath, - ) + if prompts == nil || opts.AcceptDegradationsExplicit { + return nil, fmt.Errorf( + "LibTV export reports %d explicit degradation(s); inspect %s (plan: %s), then rerun with --accept-degradations", + len(plan.Degradations), outputDir, exported.PlanPath, + ) + } + accepted, promptErr := prompts.confirmDegradations(len(plan.Degradations)) + if promptErr != nil { + return nil, promptErr + } + if !accepted { + return nil, fmt.Errorf( + "LibTV import was cancelled because the export contains %d degradation(s); inspect %s (plan: %s)", + len(plan.Degradations), outputDir, exported.PlanPath, + ) + } } media, err := readAndValidateExportMedia(exported.BundleDir, plan) if err != nil { @@ -176,7 +213,13 @@ func runCanvasImport( _ = removeOwnedBundle(outputDir, bundleRoot) return nil, err } + if err := canvasplan.PreflightJournalPath(journalPath); err != nil { + _ = removeOwnedBundle(outputDir, bundleRoot) + return nil, fmt.Errorf("preflight resolved Canvas import journal before media upload: %w", err) + } + fmt.Fprintf(stderr, "Resume journal: %s\n", journalPath) checkpointPath := journalPath + ".media.json" + fmt.Fprintf(stderr, "Phase media: resolving %d exported media file(s)...\n", len(media)) resolved, err := resolveImportMedia(ctx, mediaResolutionOptions{ Plan: plan, Media: media, @@ -189,7 +232,7 @@ func runCanvasImport( if err != nil { return nil, err } - fmt.Fprintln(stderr, "Creating or resuming the personal novel Canvas transaction...") + fmt.Fprintln(stderr, "Phase canvas: create/resume, materialize, apply, then verify remote Canvas assets.") result, executeErr := dependencies.executor.Execute(ctx, plan, resolved, canvasplan.ExecuteOptions{ JournalPath: journalPath, }) @@ -207,7 +250,7 @@ func runCanvasImport( fmt.Fprintf(stderr, "Canvas verified, but could not open the browser: %v\n", err) } } - fmt.Fprintln(stderr, "Canvas import verified.") + fmt.Fprintln(stderr, "Phase canvas: Canvas import verified by query-back.") return result, nil } diff --git a/cmd/canvas/import_journal.go b/cmd/canvas/import_journal.go new file mode 100644 index 0000000..02b4892 --- /dev/null +++ b/cmd/canvas/import_journal.go @@ -0,0 +1,64 @@ +package canvas + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/canvasplan" +) + +func preflightExplicitImportJournal(value string, explicitlySet bool) (string, error) { + trimmed := strings.TrimSpace(value) + if !explicitlySet && trimmed == "" { + return "", nil + } + if trimmed == "" { + return "", fmt.Errorf( + "canvas import --journal was explicitly set but is empty; unset/omit --journal to use the automatic journal, or first mkdir a dedicated writable directory and pass a file inside it", + ) + } + absolute, err := filepath.Abs(trimmed) + if err != nil { + return "", fmt.Errorf("resolve canvas import --journal: %w", err) + } + absolute = filepath.Clean(absolute) + parent := filepath.Dir(absolute) + if absolute == parent || filepath.Dir(parent) == parent { + return "", fmt.Errorf( + "canvas import --journal %q resolves at or directly under the filesystem root; this often means a shell directory variable was empty; unset/omit --journal to use the automatic journal, or first mkdir a dedicated writable directory and pass a file inside it", + absolute, + ) + } + if info, lstatErr := os.Lstat(absolute); lstatErr == nil { + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("canvas import --journal must not be a symbolic link: %s", absolute) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("canvas import --journal must name a regular file: %s", absolute) + } + } else if !os.IsNotExist(lstatErr) { + return "", fmt.Errorf("inspect canvas import --journal: %w", lstatErr) + } + parentInfo, err := os.Lstat(parent) + if os.IsNotExist(err) { + return "", fmt.Errorf( + "canvas import --journal parent does not exist: %s; first run mkdir -p %q, or unset/omit --journal to use the automatic journal", + parent, parent, + ) + } + if err != nil { + return "", fmt.Errorf("inspect canvas import --journal parent: %w", err) + } + if parentInfo.Mode()&os.ModeSymlink != 0 || !parentInfo.IsDir() { + return "", fmt.Errorf("canvas import --journal parent must be a real directory: %s", parent) + } + if err := canvasplan.PreflightJournalPath(absolute); err != nil { + return "", fmt.Errorf( + "canvas import --journal path is not safe and writable: %s; choose or first mkdir a writable directory, or unset/omit --journal to use the automatic journal: %w", + parent, err, + ) + } + return absolute, nil +} diff --git a/cmd/canvas/import_journal_security_unix_test.go b/cmd/canvas/import_journal_security_unix_test.go new file mode 100644 index 0000000..bed0253 --- /dev/null +++ b/cmd/canvas/import_journal_security_unix_test.go @@ -0,0 +1,86 @@ +//go:build !windows + +package canvas + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestImportCommandRejectsJournalAncestorSymlinkBeforeSideEffects(t *testing.T) { + root := t.TempDir() + realParent := filepath.Join(root, "real") + if err := os.MkdirAll(filepath.Join(realParent, "nested"), 0o700); err != nil { + t.Fatal(err) + } + linkedParent := filepath.Join(root, "linked") + if err := os.Symlink(realParent, linkedParent); err != nil { + t.Fatal(err) + } + + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &fakeImportMediaAPI{} + executor := &fakeImportExecutor{} + deps := testImportDependencies(root, exporter, media, executor) + cmd := newImportCommand(io.Discard, io.Discard, deps) + cmd.SetArgs([]string{ + "--from", "libtv", + "--url", testLibTVURL, + "--journal", filepath.Join(linkedParent, "nested", "import.journal.json"), + }) + cmd.SilenceUsage = true + + err := cmd.Execute() + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "symbolic link") { + t.Fatalf("Execute() error = %v, want ancestor symbolic-link rejection", err) + } + if len(exporter.urls) != 0 || media.uploads != 0 || executor.calls != 0 { + t.Fatalf( + "export/upload/execute side effects = %d/%d/%d, want 0/0/0", + len(exporter.urls), media.uploads, executor.calls, + ) + } +} + +func TestImportCommandChecksAutomaticJournalBeforeRemoteMediaUpload(t *testing.T) { + root := t.TempDir() + realConfig := filepath.Join(root, "real-config") + if err := os.MkdirAll(realConfig, 0o700); err != nil { + t.Fatal(err) + } + linkedConfig := filepath.Join(root, "linked-config") + if err := os.Symlink(realConfig, linkedConfig); err != nil { + t.Fatal(err) + } + + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &fakeImportMediaAPI{} + executor := &fakeImportExecutor{} + deps := testImportDependencies(root, exporter, media, executor) + deps.userConfigDir = func() (string, error) { return linkedConfig, nil } + cmd := newImportCommand(io.Discard, io.Discard, deps) + cmd.SetArgs([]string{"--from", "libtv", "--url", testLibTVURL}) + cmd.SilenceUsage = true + + err := cmd.Execute() + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "symbolic link") { + t.Fatalf("Execute() error = %v, want automatic-journal symbolic-link rejection", err) + } + if len(exporter.urls) != 1 || media.uploads != 0 || executor.calls != 0 { + t.Fatalf( + "export/upload/execute side effects = %d/%d/%d, want local export only (1/0/0)", + len(exporter.urls), media.uploads, executor.calls, + ) + } + if len(exporter.bundles) != 1 { + t.Fatalf("export bundles = %#v, want one local export", exporter.bundles) + } + if _, statErr := os.Stat(exporter.bundles[0]); !os.IsNotExist(statErr) { + t.Fatalf("unsafe automatic-journal export bundle was not removed: %v", statErr) + } +} diff --git a/cmd/canvas/import_media.go b/cmd/canvas/import_media.go index ce68c12..3aa9c9b 100644 --- a/cmd/canvas/import_media.go +++ b/cmd/canvas/import_media.go @@ -68,6 +68,7 @@ func (api runnerImportMediaAPI) PreflightUpload(ctx context.Context) error { type validatedImportMedia struct { LogicalID string MediaType string + FileName string LocalPath string SHA256 string ByteSize int64 @@ -128,6 +129,7 @@ func readAndValidateExportMedia(bundleDir string, plan canvasplan.Plan) ([]valid result = append(result, validatedImportMedia{ LogicalID: requirement.LogicalID, MediaType: requirement.MediaType, + FileName: requirement.FileName, LocalPath: localPath, SHA256: digest, ByteSize: info.Size(), @@ -179,9 +181,13 @@ func resolveImportMedia( if err != nil { return canvasplan.ResolvedMediaSet{}, err } + if len(opts.Media) == 0 { + reportImportMediaProgress(stderr, 0, 0, "complete", validatedImportMedia{FileName: "(none)"}) + } queriedReadyAssetIDs := make(map[string]struct{}) - for _, media := range opts.Media { + for index, media := range opts.Media { if existing := entries[media.LogicalID]; existing != nil { + action := "reused" switch existing.Status { case mediaStatusBlocked: return canvasplan.ResolvedMediaSet{}, fmt.Errorf( @@ -204,7 +210,7 @@ func resolveImportMedia( media.LogicalID, opts.CheckpointPath, ) case mediaStatusProcessing: - fmt.Fprintf(stderr, "Checking previously uploaded media %q...\n", media.LogicalID) + reportImportMediaProgress(stderr, index, len(opts.Media), "checking", media) if err := api.Query(ctx, existing.PippitAssetID); err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( "previous media upload %q is not queryable yet; durable IDs remain in %s: %w", @@ -217,9 +223,10 @@ func resolveImportMedia( if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, *existing); err != nil { return canvasplan.ResolvedMediaSet{}, err } + action = "queried" case mediaStatusReady: if _, queried := queriedReadyAssetIDs[existing.PippitAssetID]; !queried { - fmt.Fprintf(stderr, "Verifying previously uploaded media %q for the current Pippit account...\n", media.LogicalID) + reportImportMediaProgress(stderr, index, len(opts.Media), "checking", media) if err := api.Query(ctx, existing.PippitAssetID); err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( "previously uploaded media %q is unavailable to the current Pippit account; refusing checkpoint reuse: %w", @@ -231,11 +238,12 @@ func resolveImportMedia( default: return canvasplan.ResolvedMediaSet{}, fmt.Errorf("media checkpoint %q has invalid status %q", media.LogicalID, existing.Status) } + reportImportMediaProgress(stderr, index+1, len(opts.Media), action, media) continue } if duplicate := readyEntryByDigest(entries, media); duplicate != nil { if _, queried := queriedReadyAssetIDs[duplicate.PippitAssetID]; !queried { - fmt.Fprintf(stderr, "Verifying deduplicated media %q for the current Pippit account...\n", media.LogicalID) + reportImportMediaProgress(stderr, index, len(opts.Media), "checking", media) if err := api.Query(ctx, duplicate.PippitAssetID); err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( "deduplicated media %q is unavailable to the current Pippit account; refusing checkpoint reuse: %w", @@ -256,6 +264,7 @@ func resolveImportMedia( if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { return canvasplan.ResolvedMediaSet{}, err } + reportImportMediaProgress(stderr, index+1, len(opts.Media), "reused", media) continue } if preflighter, ok := api.(importMediaPreflighter); ok { @@ -266,7 +275,6 @@ func resolveImportMedia( ) } } - fmt.Fprintf(stderr, "Uploading LibTV media %d/%d...\n", len(entries)+1, len(opts.Media)) entry := mediaCheckpointEntry{ LogicalID: media.LogicalID, MediaType: media.MediaType, @@ -281,6 +289,7 @@ func resolveImportMedia( ) } entries[media.LogicalID] = &entry + reportImportMediaProgress(stderr, index, len(opts.Media), "uploading", media) uploaded, uploadErr := api.Upload(ctx, media.LocalPath) if uploadErr != nil && strings.Contains(uploadErr.Error(), "XYQ_ACCESS_KEY 缺失") { if checkpointErr := removeAndSaveMediaEntry(opts.CheckpointPath, checkpoint, media.LogicalID); checkpointErr != nil { @@ -334,6 +343,7 @@ func resolveImportMedia( if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { return canvasplan.ResolvedMediaSet{}, err } + reportImportMediaProgress(stderr, index+1, len(opts.Media), "uploaded", media) return canvasplan.ResolvedMediaSet{}, fmt.Errorf( "media upload %q is still processing; durable IDs are checkpointed in %s, rerun to query without re-uploading", media.LogicalID, opts.CheckpointPath, @@ -345,6 +355,7 @@ func resolveImportMedia( if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { return canvasplan.ResolvedMediaSet{}, err } + reportImportMediaProgress(stderr, index+1, len(opts.Media), "uploaded", media) } resolved := canvasplan.ResolvedMediaSet{Schema: canvasplan.ResolvedMediaSchema} for _, media := range opts.Media { @@ -364,6 +375,32 @@ func resolveImportMedia( return resolved, nil } +func reportImportMediaProgress( + stderr io.Writer, + processed int, + total int, + action string, + media validatedImportMedia, +) { + remaining := total - processed + if remaining < 0 { + remaining = 0 + } + fileName := strings.TrimSpace(media.FileName) + if fileName == "" { + fileName = filepath.Base(media.LocalPath) + } + fmt.Fprintf( + stderr, + "Media progress: processed=%d/%d remaining=%d action=%s file=%q\n", + processed, + total, + remaining, + action, + fileName, + ) +} + func loadMediaCheckpoint(opts mediaResolutionOptions) (*mediaCheckpoint, error) { if info, lstatErr := os.Lstat(opts.CheckpointPath); lstatErr == nil { if info.Mode()&os.ModeSymlink != 0 { diff --git a/cmd/canvas/import_prompt.go b/cmd/canvas/import_prompt.go new file mode 100644 index 0000000..f4e6610 --- /dev/null +++ b/cmd/canvas/import_prompt.go @@ -0,0 +1,134 @@ +package canvas + +import ( + "bufio" + "fmt" + "io" + "os" + "strings" +) + +const importFlagsHint = `--from libtv --url "https://www.liblib.tv/canvas?projectId=<project-id>"` + +type importPromptSession struct { + reader *bufio.Reader + stderr io.Writer + eof bool +} + +func importInputIsInteractive(input io.Reader) bool { + file, ok := input.(*os.File) + if !ok { + return false + } + info, err := file.Stat() + return err == nil && info.Mode()&os.ModeCharDevice != 0 +} + +func prepareCanvasImportOptions( + input io.Reader, + opts importOptions, + isInteractive func(io.Reader) bool, + stderr io.Writer, +) (importOptions, *importPromptSession, error) { + needsWizard := strings.TrimSpace(opts.Provider) == "" || strings.TrimSpace(opts.SourceURL) == "" + if !needsWizard { + return opts, nil, nil + } + if isInteractive == nil || !isInteractive(input) { + return opts, nil, fmt.Errorf( + "canvas import is missing --from or --url and stdin is not interactive; pass %s", + importFlagsHint, + ) + } + prompts := &importPromptSession{reader: bufio.NewReader(input), stderr: stderr} + if strings.TrimSpace(opts.Provider) == "" { + value, _, err := prompts.readLine("Source provider [libtv]: ") + if err != nil { + return opts, nil, err + } + if value == "" { + value = "libtv" + } + opts.Provider = value + } + if strings.TrimSpace(opts.SourceURL) == "" { + for { + value, eof, err := prompts.readLine("LibTV canvas URL: ") + if err != nil { + return opts, nil, err + } + if value != "" { + opts.SourceURL = value + break + } + if eof { + return opts, nil, fmt.Errorf( + "interactive input ended before a LibTV URL was provided; rerun with %s", + importFlagsHint, + ) + } + fmt.Fprintln(stderr, "A LibTV canvas URL is required.") + } + } + if !opts.JournalExplicit { + value, _, err := prompts.readLine("Resume journal path [automatic]: ") + if err != nil { + return opts, nil, err + } + if value != "" { + opts.JournalPath = value + opts.JournalExplicit = true + } + } + if !opts.OpenExplicit { + open, err := prompts.askYesNo("Open the imported Canvas when finished? [Y/n]: ", true) + if err != nil { + return opts, nil, err + } + opts.Open = open + } + return opts, prompts, nil +} + +func (prompts *importPromptSession) confirmDegradations(count int) (bool, error) { + fmt.Fprintf(prompts.stderr, "LibTV export reports %d explicit degradation(s).\n", count) + return prompts.askYesNo("Continue importing with these degradations? [y/N]: ", false) +} + +func (prompts *importPromptSession) askYesNo(label string, defaultValue bool) (bool, error) { + for { + value, eof, err := prompts.readLine(label) + if err != nil { + return false, err + } + switch strings.ToLower(value) { + case "": + return defaultValue, nil + case "y", "yes": + return true, nil + case "n", "no": + return false, nil + default: + if eof { + return defaultValue, nil + } + fmt.Fprintln(prompts.stderr, "Please answer y or n.") + } + } +} + +func (prompts *importPromptSession) readLine(label string) (string, bool, error) { + fmt.Fprint(prompts.stderr, label) + if prompts.eof { + return "", true, nil + } + line, err := prompts.reader.ReadString('\n') + if err != nil && err != io.EOF { + return "", false, fmt.Errorf("read canvas import prompt: %w", err) + } + if err == io.EOF { + prompts.eof = true + } + return strings.TrimSpace(line), prompts.eof, nil +} diff --git a/cmd/canvas/import_test.go b/cmd/canvas/import_test.go index c878ca6..9557ae8 100644 --- a/cmd/canvas/import_test.go +++ b/cmd/canvas/import_test.go @@ -221,6 +221,148 @@ func TestImportCommandExportsUploadsDeduplicatesVerifiesAndOpens(t *testing.T) { } } +func TestImportCommandInteractiveWizardUsesSafeDefaults(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &fakeImportMediaAPI{} + executor := &fakeImportExecutor{result: verifiedImportResult()} + opened := "" + deps := testImportDependencies(temp, exporter, media, executor) + deps.isInteractive = func(io.Reader) bool { return true } + deps.openURL = func(_ context.Context, value string) error { opened = value; return nil } + var stdout, stderr bytes.Buffer + cmd := newImportCommand(&stdout, &stderr, deps) + cmd.SetIn(strings.NewReader("\n" + testLibTVURL + "\n\n\n")) + cmd.SilenceUsage = true + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v, stderr = %s", err, stderr.String()) + } + wantURL := "https://www.liblib.tv/canvas?projectId=037a5c49e1b344e5adbc899ad93fdca9&spaceId=3872811" + if len(exporter.urls) != 1 || exporter.urls[0] != wantURL { + t.Fatalf("export URLs = %#v, want prompted LibTV URL", exporter.urls) + } + if opened != executor.result.WebURL { + t.Fatalf("opened = %q, want wizard default Yes", opened) + } + for _, message := range []string{ + "Source provider [libtv]", "LibTV canvas URL", "Resume journal path [automatic]", + "Open the imported Canvas when finished? [Y/n]", + "Resume journal: " + executor.opts.JournalPath, + `Media progress: processed=1/2 remaining=1 action=uploaded file="one.png"`, + `Media progress: processed=2/2 remaining=0 action=reused file="two.png"`, + `Media progress: processed=0/2 remaining=2 action=uploading file="one.png"`, + "Phase canvas: create/resume, materialize, apply, then verify remote Canvas assets.", + "Phase canvas: Canvas import verified by query-back.", + } { + if !strings.Contains(stderr.String(), message) { + t.Fatalf("stderr missing %q:\n%s", message, stderr.String()) + } + } + if strings.Count(stdout.String(), "\n") != 1 || !json.Valid(bytes.TrimSpace(stdout.Bytes())) { + t.Fatalf("stdout = %q, want one final JSON line", stdout.String()) + } +} + +func TestImportCommandInteractiveWizardCanAcceptDegradations(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, true) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + executor := &fakeImportExecutor{result: verifiedImportResult()} + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + deps.isInteractive = func(io.Reader) bool { return true } + var stdout, stderr bytes.Buffer + cmd := newImportCommand(&stdout, &stderr, deps) + cmd.SetIn(strings.NewReader("\n" + testLibTVURL + "\n\nn\ny\n")) + cmd.SilenceUsage = true + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v, stderr = %s", err, stderr.String()) + } + if !strings.Contains(stderr.String(), "Continue importing with these degradations? [y/N]") { + t.Fatalf("stderr = %q, want in-session degradation confirmation", stderr.String()) + } + if executor.calls != 1 || !json.Valid(bytes.TrimSpace(stdout.Bytes())) { + t.Fatalf("executor/stdout = %d/%q, want completed interactive import", executor.calls, stdout.String()) + } +} + +func TestImportCommandMissingFlagsFailsActionablyWithoutInteractiveInput(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, &fakeImportExecutor{}) + cmd := newImportCommand(io.Discard, io.Discard, deps) + cmd.SetIn(strings.NewReader("")) + cmd.SilenceUsage = true + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "stdin is not interactive") || + !strings.Contains(err.Error(), "--from libtv --url") { + t.Fatalf("Execute() error = %v, want actionable non-interactive flags", err) + } + if len(exporter.urls) != 0 { + t.Fatalf("exporter called before required input validation: %#v", exporter.urls) + } +} + +func TestImportCommandInteractiveEOFMissingURLFailsBeforeExport(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, &fakeImportExecutor{}) + deps.isInteractive = func(io.Reader) bool { return true } + cmd := newImportCommand(io.Discard, io.Discard, deps) + cmd.SetIn(strings.NewReader("\n")) + cmd.SilenceUsage = true + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "ended before a LibTV URL") || + !strings.Contains(err.Error(), "--from libtv --url") { + t.Fatalf("Execute() error = %v, want actionable EOF guidance", err) + } + if len(exporter.urls) != 0 { + t.Fatalf("exporter called after prompt EOF: %#v", exporter.urls) + } +} + +func TestImportCommandRejectsUnsafeExplicitJournalBeforeExport(t *testing.T) { + for _, test := range []struct { + name string + journal string + wantMessage string + }{ + {name: "explicit empty", journal: "", wantMessage: "explicitly set but is empty"}, + { + name: "filesystem root child", + journal: filepath.Join(string(filepath.Separator), "import.journal.json"), + wantMessage: "shell directory variable was empty", + }, + { + name: "missing parent", + journal: filepath.Join(t.TempDir(), "not-created", "import.journal.json"), + wantMessage: "mkdir -p", + }, + } { + t.Run(test.name, func(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, &fakeImportExecutor{}) + cmd := newImportCommand(io.Discard, io.Discard, deps) + cmd.SetArgs([]string{ + "--from", "libtv", "--url", testLibTVURL, "--journal", test.journal, + }) + cmd.SilenceUsage = true + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), test.wantMessage) || + !strings.Contains(err.Error(), "unset/omit --journal") { + t.Fatalf("Execute() error = %v, want early journal guidance containing %q", err, test.wantMessage) + } + if len(exporter.urls) != 0 { + t.Fatalf("exporter called before explicit journal validation: %#v", exporter.urls) + } + }) + } +} + func TestImportCommandRequiresExplicitDegradationAcceptanceAndKeepsBundle(t *testing.T) { temp := t.TempDir() plan, mediaBytes := testImportPlan(t, true) @@ -269,6 +411,45 @@ func TestImportCommandCheckpointsProcessingUploadAndDoesNotUploadAgain(t *testin if media.uploads != 1 || media.queries != 1 || executor.calls != 1 { t.Fatalf("upload/query/execute = %d/%d/%d, want 1/1/1", media.uploads, media.queries, executor.calls) } + if !strings.Contains(stderr.String(), `Media progress: processed=1/1 remaining=0 action=queried file="one.png"`) { + t.Fatalf("stderr = %q, want stable query progress", stderr.String()) + } + if !strings.Contains(stderr.String(), `Media progress: processed=0/1 remaining=1 action=checking file="one.png"`) { + t.Fatalf("stderr = %q, want pre-query progress before a potentially slow request", stderr.String()) + } +} + +func TestImportMediaProgressReportsEmptySet(t *testing.T) { + root := t.TempDir() + bundleRoot := filepath.Join(root, "exports") + opts := mediaResolutionOptions{ + Plan: canvasplan.Plan{ + Schema: canvasplan.PlanSchema, + Source: canvasplan.Source{ + Provider: "libtv", ProjectID: "037a5c49e1b344e5adbc899ad93fdca9", + Fingerprint: "sha256:" + strings.Repeat("1", 64), + }, + }, + Target: "https://xyq.jianying.com|prod", + BundleDir: filepath.Join(bundleRoot, "export-empty"), + BundleRoot: bundleRoot, + CanvasJournalPath: filepath.Join(root, "state", "canvas.journal.json"), + CheckpointPath: filepath.Join(root, "state", "canvas.journal.json.media.json"), + } + if err := os.MkdirAll(opts.BundleDir, 0o700); err != nil { + t.Fatal(err) + } + var stderr bytes.Buffer + resolved, err := resolveImportMedia(context.Background(), opts, &fakeImportMediaAPI{}, &stderr) + if err != nil { + t.Fatalf("resolveImportMedia() error = %v", err) + } + if len(resolved.Media) != 0 || !strings.Contains( + stderr.String(), + `Media progress: processed=0/0 remaining=0 action=complete file="(none)"`, + ) { + t.Fatalf("resolved/stderr = %#v/%q, want explicit 0/0 progress", resolved, stderr.String()) + } } func TestImportCommandBlocksUnknownUploadOutcome(t *testing.T) { diff --git a/internal/canvasplan/journal_preflight.go b/internal/canvasplan/journal_preflight.go new file mode 100644 index 0000000..937f3d6 --- /dev/null +++ b/internal/canvasplan/journal_preflight.go @@ -0,0 +1,43 @@ +package canvasplan + +import ( + "fmt" + "os" + "path/filepath" +) + +// PreflightJournalPath applies the same no-follow directory, journal, and lock +// checks used by Execute before a caller performs any remote side effects. +func PreflightJournalPath(path string) error { + absolute, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("resolve CanvasPlan journal preflight path: %w", err) + } + absolute = filepath.Clean(absolute) + if _, err := ensureSecureJournalDirectory(filepath.Dir(absolute)); err != nil { + return fmt.Errorf("preflight CanvasPlan journal directory: %w", err) + } + + info, err := lstatRegularOrMissing(absolute) + if err != nil { + return fmt.Errorf("inspect CanvasPlan journal during preflight: %w", err) + } + if info != nil { + file, openErr := openRegularFileNoFollow(absolute, os.O_RDWR, 0) + if openErr != nil { + return fmt.Errorf("open CanvasPlan journal during preflight: %w", openErr) + } + if closeErr := file.Close(); closeErr != nil { + return fmt.Errorf("close CanvasPlan journal after preflight: %w", closeErr) + } + } + + lock, err := acquireJournalLock(absolute) + if err != nil { + return fmt.Errorf("preflight CanvasPlan journal lock: %w", err) + } + if err := lock.release(); err != nil { + return fmt.Errorf("release CanvasPlan journal preflight lock: %w", err) + } + return nil +} From 6a9f58871e50dbc1471398107c516708886e09cd Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 19:05:40 +0800 Subject: [PATCH 21/48] feat(canvas): stream verified upload readers Co-authored-by: Codex <codex@openai.com> --- internal/canvas/canvas_test.go | 26 +++++++++++++++++++++++++ internal/canvas/upload.go | 26 ++++++++++++++++++------- internal/common/client.go | 20 ++++++++++++++----- internal/common/client_test.go | 35 ++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 12 deletions(-) diff --git a/internal/canvas/canvas_test.go b/internal/canvas/canvas_test.go index 89b5008..186c0ab 100644 --- a/internal/canvas/canvas_test.go +++ b/internal/canvas/canvas_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "strings" @@ -408,6 +409,31 @@ func TestUploadWaitsForQueryableAssetWithoutRequiringCover(t *testing.T) { } } +func TestUploadStreamsCallerVerifiedReader(t *testing.T) { + client := &fakeClient{ + multipart: func(_ context.Context, _ string, _ map[string]string, file common.MultipartFile, out any) error { + if file.Path != "" || file.FileName != "verified.png" || file.Reader == nil { + t.Fatalf("multipart file = %#v, want caller-provided reader", file) + } + payload, err := io.ReadAll(file.Reader) + if err != nil || string(payload) != "verified image" { + t.Fatalf("multipart reader payload=%q error=%v", payload, err) + } + return decodeInto(out, `{"ret":"0","data":{"asset_id":"workspace-1","pippit_asset_id":"asset-1"}}`) + }, + send: func(_ context.Context, _ string, _ any, out any) error { + return decodeInto(out, `{"ret":"0","data":{"Assets":[{"PippitAssetID":"asset-1"}]}}`) + }, + } + result, err := Upload(context.Background(), UploadOptions{ + FileName: "verified.png", + Reader: strings.NewReader("verified image"), + }, runnerWithClient(client)) + if err != nil || result.State != StateReady { + t.Fatalf("Upload() result=%#v error=%v", result, err) + } +} + func TestUploadWaitTimeoutPreservesDurableAssetID(t *testing.T) { path := filepath.Join(t.TempDir(), "clip.mp4") if err := os.WriteFile(path, []byte("video"), 0o600); err != nil { diff --git a/internal/canvas/upload.go b/internal/canvas/upload.go index 482f271..b408c16 100644 --- a/internal/canvas/upload.go +++ b/internal/canvas/upload.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "mime" "os" "path/filepath" @@ -29,6 +30,8 @@ var uploadContentTypeFallbacks = map[string]string{ type UploadOptions struct { Path string + FileName string + Reader io.Reader PollInterval time.Duration WaitTimeout time.Duration } @@ -66,17 +69,25 @@ func Upload(ctx context.Context, opts UploadOptions, runner *common.Runner) (*Up return nil, fmt.Errorf("canvas upload polling durations must not be negative") } path := strings.TrimSpace(opts.Path) - if path == "" { + if path == "" && opts.Reader == nil { return nil, fmt.Errorf("canvas upload path is required") } - info, err := os.Stat(path) - if err != nil { - return nil, fmt.Errorf("inspect canvas upload file: %w", err) + if opts.Reader == nil { + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("inspect canvas upload file: %w", err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("canvas upload path %q is not a regular file", path) + } + } + fileName := strings.TrimSpace(opts.FileName) + if fileName == "" { + fileName = filepath.Base(path) } - if !info.Mode().IsRegular() { - return nil, fmt.Errorf("canvas upload path %q is not a regular file", path) + if fileName == "" || fileName == "." { + return nil, fmt.Errorf("canvas upload file name is required") } - fileName := filepath.Base(path) extension := strings.ToLower(filepath.Ext(fileName)) contentType := mime.TypeByExtension(extension) if contentType == "" { @@ -92,6 +103,7 @@ func Upload(ctx context.Context, opts UploadOptions, runner *common.Runner) (*Up Path: path, FileName: fileName, ContentType: contentType, + Reader: opts.Reader, }, &envelope); err != nil { return nil, fmt.Errorf("canvas upload request failed; outcome may be ambiguous, check assets before retrying: %w", err) } diff --git a/internal/common/client.go b/internal/common/client.go index e4a46b1..65b686c 100644 --- a/internal/common/client.go +++ b/internal/common/client.go @@ -41,6 +41,7 @@ type MultipartFile struct { Path string FileName string ContentType string + Reader io.Reader } type httpClient struct { @@ -146,6 +147,9 @@ func (c *httpClient) SendMultipartRequest(ctx context.Context, path string, fiel if file.FileName == "" { file.FileName = filepath.Base(file.Path) } + if strings.TrimSpace(file.FileName) == "" || file.FileName == "." { + return fmt.Errorf("multipart 文件名不能为空") + } if file.ContentType == "" { file.ContentType = "application/octet-stream" } @@ -187,11 +191,17 @@ func writeMultipartBody(writer *multipart.Writer, fields map[string]string, file } } - f, err := os.Open(file.Path) - if err != nil { - return fmt.Errorf("打开上传文件失败: %w", err) + reader := file.Reader + var opened *os.File + if reader == nil { + var err error + opened, err = os.Open(file.Path) + if err != nil { + return fmt.Errorf("打开上传文件失败: %w", err) + } + defer opened.Close() + reader = opened } - defer f.Close() header := make(textproto.MIMEHeader) header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes(file.FieldName), escapeQuotes(file.FileName))) @@ -200,7 +210,7 @@ func writeMultipartBody(writer *multipart.Writer, fields map[string]string, file if err != nil { return err } - if _, err := io.Copy(part, f); err != nil { + if _, err := io.Copy(part, reader); err != nil { return fmt.Errorf("写入上传文件失败: %w", err) } return nil diff --git a/internal/common/client_test.go b/internal/common/client_test.go index a9dcb13..354f2dc 100644 --- a/internal/common/client_test.go +++ b/internal/common/client_test.go @@ -1,7 +1,9 @@ package common import ( + "bytes" "context" + "io" "net/http" "net/http/httptest" "net/url" @@ -263,6 +265,39 @@ func TestHTTPClientMultipartRequestInjectsPPEHeaders(t *testing.T) { } } +func TestHTTPClientMultipartRequestStreamsProvidedReader(t *testing.T) { + contents := make(chan string, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + file, _, err := r.FormFile("file") + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer file.Close() + payload, err := io.ReadAll(file) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + contents <- string(payload) + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + + client := NewHTTPClient(server.URL, time.Second, NewAccessKeyAuthorizer("reader-ak")) + err := client.SendMultipartRequest(context.Background(), "/api/upload", nil, MultipartFile{ + FieldName: "file", + FileName: "verified.bin", + Reader: bytes.NewBufferString("verified bytes"), + }, nil) + if err != nil { + t.Fatalf("SendMultipartRequest() error = %v", err) + } + if got := <-contents; got != "verified bytes" { + t.Fatalf("multipart contents = %q", got) + } +} + func TestSameOriginUsesEffectiveDefaultPorts(t *testing.T) { httpsDefault, _ := url.Parse("https://xyq.jianying.com/api") httpsExplicit, _ := url.Parse("https://XYQ.JIANYING.COM:443/asset") From 07a715fcdafdd3a0fecab365a3f4fbc54b6e1dfb Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 19:08:31 +0800 Subject: [PATCH 22/48] feat(canvas): fingerprint exported media safely Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_media_file_unix.go | 40 ++ cmd/canvas/import_media_file_windows.go | 70 +++ cmd/canvas/import_media_fingerprint.go | 732 ++++++++++++++++++++++++ 3 files changed, 842 insertions(+) create mode 100644 cmd/canvas/import_media_file_unix.go create mode 100644 cmd/canvas/import_media_file_windows.go create mode 100644 cmd/canvas/import_media_fingerprint.go diff --git a/cmd/canvas/import_media_file_unix.go b/cmd/canvas/import_media_file_unix.go new file mode 100644 index 0000000..d7225b5 --- /dev/null +++ b/cmd/canvas/import_media_file_unix.go @@ -0,0 +1,40 @@ +//go:build !windows + +package canvas + +import ( + "fmt" + "os" + "syscall" +) + +func fileInfoIsImportMediaLinkLike(info os.FileInfo) bool { + return info != nil && info.Mode()&os.ModeSymlink != 0 +} + +func openImportMediaNoFollow(path string) (*os.File, error) { + before, err := os.Lstat(path) + if err != nil { + return nil, err + } + if fileInfoIsImportMediaLinkLike(before) || !before.Mode().IsRegular() { + return nil, fmt.Errorf("import media must be a regular non-symbolic file: %s", path) + } + fd, err := syscall.Open(path, syscall.O_RDONLY|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), path) + opened, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, err + } + current, err := os.Lstat(path) + if err != nil || fileInfoIsImportMediaLinkLike(current) || !current.Mode().IsRegular() || + !opened.Mode().IsRegular() || !os.SameFile(before, opened) || !os.SameFile(opened, current) { + _ = file.Close() + return nil, fmt.Errorf("import media path changed while it was opened: %s", path) + } + return file, nil +} diff --git a/cmd/canvas/import_media_file_windows.go b/cmd/canvas/import_media_file_windows.go new file mode 100644 index 0000000..7c2fef0 --- /dev/null +++ b/cmd/canvas/import_media_file_windows.go @@ -0,0 +1,70 @@ +//go:build windows + +package canvas + +import ( + "fmt" + "os" + "syscall" + + "golang.org/x/sys/windows" +) + +func fileInfoIsImportMediaLinkLike(info os.FileInfo) bool { + if info == nil { + return false + } + if info.Mode()&os.ModeSymlink != 0 { + return true + } + data, ok := info.Sys().(*syscall.Win32FileAttributeData) + return ok && data.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 +} + +func openImportMediaNoFollow(path string) (*os.File, error) { + before, err := os.Lstat(path) + if err != nil { + return nil, err + } + if fileInfoIsImportMediaLinkLike(before) || !before.Mode().IsRegular() { + return nil, fmt.Errorf("import media must be a regular non-reparse file: %s", path) + } + pathPtr, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err + } + handle, err := windows.CreateFile( + pathPtr, + windows.GENERIC_READ, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + nil, + windows.OPEN_EXISTING, + windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_OPEN_REPARSE_POINT, + 0, + ) + if err != nil { + return nil, err + } + var handleInfo windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &handleInfo); err != nil { + _ = windows.CloseHandle(handle) + return nil, err + } + if handleInfo.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + _ = windows.CloseHandle(handle) + return nil, fmt.Errorf("import media must not be a Windows reparse point: %s", path) + } + file := os.NewFile(uintptr(handle), path) + opened, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, err + } + current, err := os.Lstat(path) + if err != nil || fileInfoIsImportMediaLinkLike(current) || !current.Mode().IsRegular() || + !opened.Mode().IsRegular() || !os.SameFile(before, opened) || !os.SameFile(opened, current) { + _ = file.Close() + return nil, fmt.Errorf("import media path changed while it was opened: %s", path) + } + return file, nil +} diff --git a/cmd/canvas/import_media_fingerprint.go b/cmd/canvas/import_media_fingerprint.go new file mode 100644 index 0000000..c544bb4 --- /dev/null +++ b/cmd/canvas/import_media_fingerprint.go @@ -0,0 +1,732 @@ +package canvas + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "hash" + "hash/crc32" + "io" + "os" + "path/filepath" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/canvasplan" +) + +const ( + libTVPNGAIGCFingerprintPrefix = "libtv-png-aigc-v1:" + rawMediaFingerprintPrefix = "raw-sha256:" + maxCanonicalITXtChunkBytes = 1 << 20 + libTVAIGCProducer = "001191110105MACJ6K1C8A10001" +) + +var pngFileSignature = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'} + +type importMediaFileIdentity struct { + RawSHA256 string + ContentFingerprint string + ByteSize int64 +} + +type legacyCheckpointMediaIdentity struct { + ContentFingerprint string + ByteSize int64 +} + +type importMediaCountingReader struct { + reader io.Reader + count int64 +} + +func (reader *importMediaCountingReader) Read(payload []byte) (int, error) { + count, err := reader.reader.Read(payload) + reader.count += int64(count) + return count, err +} + +// inspectImportMediaFile derives the byte size, raw digest, and the narrowly +// canonicalized LibTV PNG identity from one no-follow file descriptor. The +// path is checked against the descriptor before and after the stream is read, +// so callers never combine facts derived from different path targets. +func inspectImportMediaFile(path string) (importMediaFileIdentity, error) { + file, identity, err := openInspectedImportMediaFile(path) + if file != nil { + _ = file.Close() + } + return identity, err +} + +// openInspectedImportMediaFile leaves the verified descriptor open for the +// caller. This lets the upload path seek and stream the exact inode whose +// digest and canonical fingerprint were checked, without reopening by path. +func openInspectedImportMediaFile(path string) (*os.File, importMediaFileIdentity, error) { + file, err := openImportMediaNoFollow(path) + if err != nil { + return nil, importMediaFileIdentity{}, err + } + initial, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, importMediaFileIdentity{}, fmt.Errorf("inspect opened import media: %w", err) + } + identity, inspectErr := inspectImportMediaContent(file) + stableErr := validateStableImportMediaFile(path, file, initial) + if inspectErr != nil { + _ = file.Close() + return nil, importMediaFileIdentity{}, inspectErr + } + if stableErr != nil { + _ = file.Close() + return nil, importMediaFileIdentity{}, stableErr + } + if identity.ByteSize != initial.Size() { + _ = file.Close() + return nil, importMediaFileIdentity{}, fmt.Errorf("import media size changed while it was inspected") + } + return file, identity, nil +} + +func validateStableImportMediaFile(path string, file *os.File, initial os.FileInfo) error { + opened, err := file.Stat() + if err != nil { + return fmt.Errorf("reinspect opened import media: %w", err) + } + current, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("reinspect import media path: %w", err) + } + if fileInfoIsImportMediaLinkLike(current) || !current.Mode().IsRegular() || + !opened.Mode().IsRegular() || !os.SameFile(initial, opened) || !os.SameFile(opened, current) { + return fmt.Errorf("import media path changed while it was inspected: %s", path) + } + if initial.Size() != opened.Size() || !initial.ModTime().Equal(opened.ModTime()) { + return fmt.Errorf("import media content changed while it was inspected: %s", path) + } + return nil +} + +func inspectImportMediaContent(reader io.Reader) (importMediaFileIdentity, error) { + counting := &importMediaCountingReader{reader: reader} + rawHash := sha256.New() + stream := io.TeeReader(counting, rawHash) + signature := make([]byte, len(pngFileSignature)) + read, err := io.ReadFull(stream, signature) + if err != nil { + if err != io.EOF && err != io.ErrUnexpectedEOF { + return importMediaFileIdentity{}, fmt.Errorf("read import media signature: %w", err) + } + return rawImportMediaIdentity(rawHash, counting.count), nil + } + if read != len(pngFileSignature) || !bytes.Equal(signature, pngFileSignature) { + if _, err := io.Copy(io.Discard, stream); err != nil { + return importMediaFileIdentity{}, fmt.Errorf("hash import media: %w", err) + } + return rawImportMediaIdentity(rawHash, counting.count), nil + } + canonicalDigest, normalizedAIGC, err := canonicalLibTVPNGStream(stream) + if err != nil { + return importMediaFileIdentity{}, err + } + rawDigest := hex.EncodeToString(rawHash.Sum(nil)) + fingerprint := rawMediaFingerprintPrefix + rawDigest + if normalizedAIGC { + fingerprint = libTVPNGAIGCFingerprintPrefix + canonicalDigest + } + return importMediaFileIdentity{ + RawSHA256: rawDigest, + ContentFingerprint: fingerprint, + ByteSize: counting.count, + }, nil +} + +func rawImportMediaIdentity(rawHash hash.Hash, byteSize int64) importMediaFileIdentity { + digest := hex.EncodeToString(rawHash.Sum(nil)) + return importMediaFileIdentity{ + RawSHA256: digest, + ContentFingerprint: rawMediaFingerprintPrefix + digest, + ByteSize: byteSize, + } +} + +func importMediaContentFingerprint(path, rawSHA256 string) (string, error) { + identity, err := inspectImportMediaFile(path) + if err != nil { + return "", err + } + if identity.RawSHA256 != rawSHA256 { + return "", fmt.Errorf("import media raw SHA-256 changed before fingerprinting") + } + return identity.ContentFingerprint, nil +} + +func canonicalLibTVPNGFingerprint(payload []byte) (string, error) { + identity, err := inspectImportMediaContent(bytes.NewReader(payload)) + if err != nil { + return "", err + } + if len(payload) < len(pngFileSignature) || !bytes.Equal(payload[:len(pngFileSignature)], pngFileSignature) { + return "", fmt.Errorf("PNG signature is invalid") + } + return identity.ContentFingerprint, nil +} + +func canonicalLibTVPNGStream(reader io.Reader) (string, bool, error) { + canonicalHash := sha256.New() + _, _ = canonicalHash.Write(pngFileSignature) + chunkIndex := 0 + seenIHDR := false + seenPLTE := false + seenIDAT := false + endedIDAT := false + colorType := byte(0xff) + aigcChunkCount := 0 + normalizedAIGCCount := 0 + for { + var header [8]byte + if _, err := io.ReadFull(reader, header[:]); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return "", false, fmt.Errorf("PNG is missing IEND or has a truncated chunk header") + } + return "", false, fmt.Errorf("read PNG chunk header: %w", err) + } + length := uint64(binary.BigEndian.Uint32(header[:4])) + chunkType := header[4:] + chunkName := string(chunkType) + if err := validatePNGChunkType(chunkType); err != nil { + return "", false, err + } + if chunkIndex == 0 && chunkName != "IHDR" { + return "", false, fmt.Errorf("PNG IHDR must be the first chunk") + } + if chunkName != "IDAT" && seenIDAT { + endedIDAT = true + } + switch chunkName { + case "IHDR": + if seenIHDR || chunkIndex != 0 || length != 13 { + return "", false, fmt.Errorf("PNG must contain one 13-byte IHDR as its first chunk") + } + seenIHDR = true + case "PLTE": + if seenPLTE || seenIDAT || length == 0 || length > 768 || length%3 != 0 { + return "", false, fmt.Errorf("PNG has an invalid or misplaced PLTE chunk") + } + seenPLTE = true + case "IDAT": + if !seenIHDR || endedIDAT { + return "", false, fmt.Errorf("PNG IDAT chunks must be consecutive and follow IHDR") + } + if colorType == 3 && !seenPLTE { + return "", false, fmt.Errorf("indexed PNG is missing PLTE before IDAT") + } + seenIDAT = true + case "IEND": + if !seenIDAT || length != 0 { + return "", false, fmt.Errorf("PNG IEND must be empty and follow IDAT") + } + default: + if chunkType[0]&0x20 == 0 { + return "", false, fmt.Errorf("PNG contains unknown critical chunk %q", chunkName) + } + } + + bufferChunk := chunkName == "IHDR" || (chunkName == "iTXt" && length <= maxCanonicalITXtChunkBytes) + if bufferChunk { + chunkData := make([]byte, int(length)) + if _, err := io.ReadFull(reader, chunkData); err != nil { + return "", false, fmt.Errorf("read PNG chunk %q: %w", chunkName, err) + } + var crcBytes [4]byte + if _, err := io.ReadFull(reader, crcBytes[:]); err != nil { + return "", false, fmt.Errorf("read PNG chunk %q CRC: %w", chunkName, err) + } + if err := validatePNGChunkCRC(chunkType, chunkData, crcBytes); err != nil { + return "", false, err + } + if chunkName == "IHDR" { + var err error + colorType, err = validatePNGIHDR(chunkData) + if err != nil { + return "", false, err + } + } + canonicalData := chunkData + matched := false + if chunkName == "iTXt" { + if isLibTVAIGCITXt(chunkData) { + aigcChunkCount++ + } + canonicalData, matched = normalizeLibTVAIGCITXt(chunkData) + if matched { + normalizedAIGCCount++ + } + } + writeCanonicalPNGChunk(canonicalHash, chunkType, canonicalData) + } else { + _, _ = canonicalHash.Write(header[:]) + crc := crc32.NewIEEE() + _, _ = crc.Write(chunkType) + chunkWriter := io.MultiWriter(canonicalHash, crc) + remaining := int64(length) + if chunkName == "iTXt" { + prefixSize := int64(len("AIGC\x00")) + if remaining < prefixSize { + prefixSize = remaining + } + prefix := make([]byte, int(prefixSize)) + if _, err := io.ReadFull(reader, prefix); err != nil { + return "", false, fmt.Errorf("read PNG chunk %q prefix: %w", chunkName, err) + } + _, _ = chunkWriter.Write(prefix) + remaining -= prefixSize + if bytes.Equal(prefix, []byte("AIGC\x00")) { + aigcChunkCount++ + } + } + if _, err := io.CopyN(chunkWriter, reader, remaining); err != nil { + return "", false, fmt.Errorf("read PNG chunk %q: %w", chunkName, err) + } + var crcBytes [4]byte + if _, err := io.ReadFull(reader, crcBytes[:]); err != nil { + return "", false, fmt.Errorf("read PNG chunk %q CRC: %w", chunkName, err) + } + if crc.Sum32() != binary.BigEndian.Uint32(crcBytes[:]) { + return "", false, fmt.Errorf("PNG chunk %q has an invalid CRC", chunkName) + } + _, _ = canonicalHash.Write(crcBytes[:]) + } + chunkIndex++ + if chunkName == "IEND" { + var trailing [1]byte + if _, err := io.ReadFull(reader, trailing[:]); err == nil { + return "", false, fmt.Errorf("PNG contains trailing bytes after IEND") + } else if err != io.EOF { + return "", false, fmt.Errorf("inspect PNG trailing bytes: %w", err) + } + break + } + } + if !seenIHDR || !seenIDAT { + return "", false, fmt.Errorf("PNG is missing IHDR or IDAT") + } + if (colorType == 0 || colorType == 4) && seenPLTE { + return "", false, fmt.Errorf("grayscale PNG must not contain PLTE") + } + return hex.EncodeToString(canonicalHash.Sum(nil)), aigcChunkCount == 1 && normalizedAIGCCount == 1, nil +} + +func validatePNGChunkType(chunkType []byte) error { + for _, value := range chunkType { + if (value < 'A' || value > 'Z') && (value < 'a' || value > 'z') { + return fmt.Errorf("PNG chunk type %q is invalid", string(chunkType)) + } + } + if chunkType[2]&0x20 != 0 { + return fmt.Errorf("PNG chunk type %q has an invalid reserved bit", string(chunkType)) + } + return nil +} + +func validatePNGChunkCRC(chunkType, data []byte, crcBytes [4]byte) error { + crc := crc32.NewIEEE() + _, _ = crc.Write(chunkType) + _, _ = crc.Write(data) + if crc.Sum32() != binary.BigEndian.Uint32(crcBytes[:]) { + return fmt.Errorf("PNG chunk %q has an invalid CRC", string(chunkType)) + } + return nil +} + +func validatePNGIHDR(data []byte) (byte, error) { + width := binary.BigEndian.Uint32(data[:4]) + height := binary.BigEndian.Uint32(data[4:8]) + bitDepth := data[8] + colorType := data[9] + validDepth := map[byte]map[byte]bool{ + 0: {1: true, 2: true, 4: true, 8: true, 16: true}, + 2: {8: true, 16: true}, + 3: {1: true, 2: true, 4: true, 8: true}, + 4: {8: true, 16: true}, + 6: {8: true, 16: true}, + } + if width == 0 || height == 0 || !validDepth[colorType][bitDepth] || + data[10] != 0 || data[11] != 0 || data[12] > 1 { + return 0, fmt.Errorf("PNG IHDR contains invalid dimensions or encoding fields") + } + return colorType, nil +} + +func writeCanonicalPNGChunk(destination hash.Hash, chunkType, data []byte) { + var length [4]byte + binary.BigEndian.PutUint32(length[:], uint32(len(data))) + _, _ = destination.Write(length[:]) + _, _ = destination.Write(chunkType) + _, _ = destination.Write(data) + crc := crc32.NewIEEE() + _, _ = crc.Write(chunkType) + _, _ = crc.Write(data) + var checksum [4]byte + binary.BigEndian.PutUint32(checksum[:], crc.Sum32()) + _, _ = destination.Write(checksum[:]) +} + +// normalizeLibTVAIGCITXt only recognizes the observed uncompressed LibTV +// schema: keyword AIGC, empty language fields, and its exact seven string +// fields. Any old, malformed, compressed, or future schema is kept byte-for- +// byte in the fingerprint instead of blocking an otherwise valid PNG. +func normalizeLibTVAIGCITXt(data []byte) ([]byte, bool) { + header := []byte("AIGC\x00\x00\x00\x00\x00") + if !bytes.HasPrefix(data, header) { + return data, false + } + normalizedJSON, ok := normalizeLibTVAIGCJSON(data[len(header):]) + if !ok { + return data, false + } + result := make([]byte, 0, len(header)+len(normalizedJSON)) + result = append(result, header...) + result = append(result, normalizedJSON...) + return result, true +} + +func isLibTVAIGCITXt(data []byte) bool { + return bytes.HasPrefix(data, []byte("AIGC\x00")) +} + +func normalizeLibTVAIGCJSON(payload []byte) ([]byte, bool) { + if !json.Valid(payload) { + return payload, false + } + position := skipJSONWhitespace(payload, 0) + if position >= len(payload) || payload[position] != '{' { + return payload, false + } + position++ + cursor := 0 + fields := make(map[string]string, 7) + var normalized bytes.Buffer + for { + position = skipJSONWhitespace(payload, position) + if position >= len(payload) || payload[position] == '}' { + break + } + keyStart := position + keyEnd, err := scanJSONStringEnd(payload, keyStart) + if err != nil { + return payload, false + } + var key string + if err := json.Unmarshal(payload[keyStart:keyEnd], &key); err != nil { + return payload, false + } + if _, duplicate := fields[key]; duplicate || !knownLibTVAIGCField(key) { + return payload, false + } + position = skipJSONWhitespace(payload, keyEnd) + if position >= len(payload) || payload[position] != ':' { + return payload, false + } + valueStart := skipJSONWhitespace(payload, position+1) + valueEnd, err := scanJSONValueEnd(payload, valueStart) + if err != nil { + return payload, false + } + if valueStart >= len(payload) || payload[valueStart] != '"' { + return payload, false + } + var value string + if err := json.Unmarshal(payload[valueStart:valueEnd], &value); err != nil { + return payload, false + } + fields[key] = value + if key == "ProduceID" || key == "PropagateID" { + normalized.Write(payload[cursor:valueStart]) + if key == "ProduceID" { + normalized.WriteString(`"__pippit_normalized_produce_id__"`) + } else { + normalized.WriteString(`"__pippit_normalized_propagate_id__"`) + } + cursor = valueEnd + } + position = skipJSONWhitespace(payload, valueEnd) + if position < len(payload) && payload[position] == ',' { + position++ + continue + } + if position < len(payload) && payload[position] == '}' { + break + } + return payload, false + } + if len(fields) != 7 || fields["Label"] != "1" || + fields["ContentProducer"] != libTVAIGCProducer || + fields["ContentPropagator"] != libTVAIGCProducer || + fields["ReservedCode1"] != "" || fields["ReservedCode2"] != "" || + fields["ProduceID"] != fields["PropagateID"] || !validLibTVAIGCID(fields["ProduceID"]) { + return payload, false + } + normalized.Write(payload[cursor:]) + return normalized.Bytes(), true +} + +func knownLibTVAIGCField(key string) bool { + switch key { + case "Label", "ContentProducer", "ProduceID", "ReservedCode1", + "ContentPropagator", "PropagateID", "ReservedCode2": + return true + default: + return false + } +} + +func validLibTVAIGCID(value string) bool { + if len(value) != len("libtv")+32 || !strings.HasPrefix(value, "libtv") { + return false + } + for _, character := range value[len("libtv"):] { + if (character < '0' || character > '9') && (character < 'a' || character > 'f') { + return false + } + } + return true +} + +func scanJSONStringEnd(payload []byte, start int) (int, error) { + if start >= len(payload) || payload[start] != '"' { + return 0, fmt.Errorf("expected a JSON string") + } + for position := start + 1; position < len(payload); position++ { + switch payload[position] { + case '\\': + position++ + if position >= len(payload) { + return 0, fmt.Errorf("JSON string has a truncated escape") + } + case '"': + return position + 1, nil + } + } + return 0, fmt.Errorf("JSON string is unterminated") +} + +func scanJSONValueEnd(payload []byte, start int) (int, error) { + if start >= len(payload) { + return 0, fmt.Errorf("JSON value is missing") + } + if payload[start] == '"' { + return scanJSONStringEnd(payload, start) + } + if payload[start] == '{' || payload[start] == '[' { + stack := []byte{matchingJSONDelimiter(payload[start])} + for position := start + 1; position < len(payload); position++ { + if payload[position] == '"' { + end, err := scanJSONStringEnd(payload, position) + if err != nil { + return 0, err + } + position = end - 1 + continue + } + switch payload[position] { + case '{', '[': + stack = append(stack, matchingJSONDelimiter(payload[position])) + case '}', ']': + if len(stack) == 0 || payload[position] != stack[len(stack)-1] { + return 0, fmt.Errorf("JSON has mismatched delimiters") + } + stack = stack[:len(stack)-1] + if len(stack) == 0 { + return position + 1, nil + } + } + } + return 0, fmt.Errorf("JSON nested value is unterminated") + } + position := start + for position < len(payload) && payload[position] != ',' && payload[position] != '}' && + payload[position] != ']' && !isJSONWhitespace(payload[position]) { + position++ + } + if position == start { + return 0, fmt.Errorf("JSON primitive value is empty") + } + return position, nil +} + +func matchingJSONDelimiter(value byte) byte { + if value == '{' { + return '}' + } + return ']' +} + +func skipJSONWhitespace(payload []byte, position int) int { + for position < len(payload) && isJSONWhitespace(payload[position]) { + position++ + } + return position +} + +func isJSONWhitespace(value byte) bool { + return value == ' ' || value == '\t' || value == '\r' || value == '\n' +} + +func findLegacyCheckpointImageIdentity( + checkpoint *mediaCheckpoint, + opts mediaResolutionOptions, + entry mediaCheckpointEntry, +) (legacyCheckpointMediaIdentity, error) { + current := findMediaRequirement(opts.Plan.RequiredMedia, entry.LogicalID) + if current == nil || current.MediaType != "image" || current.URL != "" || current.LocalPath == "" { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("current checkpoint media requirement is not a local image") + } + candidates := make([]legacyCheckpointMediaIdentity, 0, 1) + for _, bundleDir := range checkpoint.BundleDirs { + if err := validateOwnedImportBundle(bundleDir, opts.BundleRoot); err != nil { + if os.IsNotExist(err) { + continue + } + return legacyCheckpointMediaIdentity{}, err + } + planPath := filepath.Join(bundleDir, "plan.json") + plan, err := readLegacyCanvasPlanNoFollow(planPath) + if os.IsNotExist(err) { + continue + } + if err != nil { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("read previous CanvasPlan: %w", err) + } + if plan.Source != checkpoint.Source { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("previous CanvasPlan source does not match the checkpoint") + } + previous := findMediaRequirement(plan.RequiredMedia, entry.LogicalID) + if previous == nil || previous.SHA256 != entry.SHA256 { + continue + } + if !sameLegacyMediaRequirement(*previous, *current) { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("previous checkpoint media requirement does not match the current source node and media contract") + } + previousPath := filepath.Join(bundleDir, filepath.FromSlash(previous.LocalPath)) + if err := requireFileWithinBundle(previousPath, bundleDir); err != nil { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("validate previous checkpoint image path: %w", err) + } + identity, err := inspectImportMediaFile(previousPath) + if err != nil { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("inspect previous checkpoint image: %w", err) + } + if previous.Metadata.ByteSize == nil || identity.ByteSize != *previous.Metadata.ByteSize { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("previous checkpoint image byte size does not match its CanvasPlan") + } + if identity.RawSHA256 != entry.SHA256 { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("previous checkpoint image raw SHA-256 does not match the checkpoint") + } + candidates = append(candidates, legacyCheckpointMediaIdentity{ + ContentFingerprint: identity.ContentFingerprint, + ByteSize: identity.ByteSize, + }) + } + if len(candidates) == 0 { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("no retained export bundle contains the checkpointed raw image") + } + if len(candidates) != 1 { + return legacyCheckpointMediaIdentity{}, fmt.Errorf("multiple retained export bundles ambiguously match the checkpointed raw image") + } + return candidates[0], nil +} + +func readLegacyCanvasPlanNoFollow(path string) (canvasplan.Plan, error) { + file, err := openImportMediaNoFollow(path) + if err != nil { + return canvasplan.Plan{}, err + } + defer file.Close() + initial, err := file.Stat() + if err != nil { + return canvasplan.Plan{}, err + } + plan, decodeErr := canvasplan.DecodePlan(file) + stableErr := validateStableImportMediaFile(path, file, initial) + if decodeErr != nil { + return canvasplan.Plan{}, decodeErr + } + if stableErr != nil { + return canvasplan.Plan{}, stableErr + } + return plan, nil +} + +func findMediaRequirement(requirements []canvasplan.MediaRequirement, logicalID string) *canvasplan.MediaRequirement { + for index := range requirements { + if requirements[index].LogicalID == logicalID { + return &requirements[index] + } + } + return nil +} + +func sameLegacyMediaRequirement(previous, current canvasplan.MediaRequirement) bool { + return previous.LogicalID == current.LogicalID && + previous.SourceNodeID == current.SourceNodeID && + previous.FileName == current.FileName && + previous.MediaType == current.MediaType && + previous.URL == current.URL && + previous.LocalPath == current.LocalPath && + sameMediaMetadata(previous.Metadata, current.Metadata) +} + +func sameMediaMetadata(previous, current canvasplan.MediaMetadata) bool { + return optionalInt64Equal(previous.ByteSize, current.ByteSize) && + optionalInt64Equal(previous.DurationMS, current.DurationMS) && + previous.Extension == current.Extension && + optionalInt64Equal(previous.Height, current.Height) && + previous.MimeType == current.MimeType && + optionalInt64Equal(previous.Width, current.Width) +} + +func optionalInt64Equal(left, right *int64) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return *left == *right +} + +func validateOwnedImportBundle(bundleDir, bundleRoot string) error { + root, err := filepath.Abs(bundleRoot) + if err != nil { + return fmt.Errorf("resolve canvas import bundle root: %w", err) + } + bundle, err := filepath.Abs(bundleDir) + if err != nil { + return fmt.Errorf("resolve checkpoint bundle: %w", err) + } + relative, err := filepath.Rel(root, bundle) + if err != nil || strings.Contains(relative, string(filepath.Separator)) || !strings.HasPrefix(relative, "export-") { + return fmt.Errorf("checkpoint bundle is outside the owned import bundle root") + } + info, err := os.Lstat(bundle) + if err != nil { + return fmt.Errorf("inspect checkpoint bundle: %w", err) + } + if fileInfoIsImportMediaLinkLike(info) || !info.IsDir() { + return fmt.Errorf("checkpoint bundle must be a real directory") + } + return nil +} + +func validImportMediaContentFingerprint(value string) bool { + for _, prefix := range []string{libTVPNGAIGCFingerprintPrefix, rawMediaFingerprintPrefix} { + if strings.HasPrefix(value, prefix) { + digest := strings.TrimPrefix(value, prefix) + if len(digest) != sha256.Size*2 || strings.ToLower(digest) != digest { + return false + } + _, err := hex.DecodeString(digest) + return err == nil + } + } + return false +} From 59682af3aee952749b9e503e60e5ff80af31b721 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 19:09:07 +0800 Subject: [PATCH 23/48] fix(canvas): resume media uploads safely Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_media.go | 353 +++++++++++++++++++++++++++++-------- 1 file changed, 275 insertions(+), 78 deletions(-) diff --git a/cmd/canvas/import_media.go b/cmd/canvas/import_media.go index 3aa9c9b..6afbc6a 100644 --- a/cmd/canvas/import_media.go +++ b/cmd/canvas/import_media.go @@ -11,6 +11,7 @@ import ( "path/filepath" "sort" "strings" + "time" canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" "github.com/Pippit-dev/pippit-cli/internal/canvasplan" @@ -25,6 +26,9 @@ const ( mediaStatusBlocked = "blocked" mediaStatusBlockedInterruption = "blocked-on-interruption" maxMediaCheckpointBytes = 8 << 20 + defaultImportMediaPollInterval = 2 * time.Second + defaultImportMediaWaitTimeout = 10 * time.Minute + initialImportUploadWaitTimeout = 5 * time.Second ) type importMediaPreflighter interface { @@ -32,21 +36,41 @@ type importMediaPreflighter interface { } type importMediaAPI interface { - Upload(context.Context, string) (*canvascore.UploadResult, error) - Query(context.Context, string) error + Upload(context.Context, validatedImportMedia) (*canvascore.UploadResult, error) + Query(context.Context, string) (bool, error) } type runnerImportMediaAPI struct { runner *common.Runner } -func (api runnerImportMediaAPI) Upload(ctx context.Context, path string) (*canvascore.UploadResult, error) { - return canvascore.Upload(ctx, canvascore.UploadOptions{Path: path}, api.runner) +func (api runnerImportMediaAPI) Upload(ctx context.Context, media validatedImportMedia) (*canvascore.UploadResult, error) { + file, identity, err := openInspectedImportMediaFile(media.LocalPath) + if err != nil { + return nil, fmt.Errorf("verify canvas import media immediately before upload: %w", err) + } + defer file.Close() + if identity.RawSHA256 != media.SHA256 || identity.ContentFingerprint != media.ContentFingerprint || + identity.ByteSize != media.ByteSize { + return nil, fmt.Errorf("canvas import media changed before upload dispatch") + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, fmt.Errorf("rewind verified canvas import media before upload: %w", err) + } + return canvascore.Upload(ctx, canvascore.UploadOptions{ + FileName: media.FileName, + Reader: file, + PollInterval: time.Second, + WaitTimeout: initialImportUploadWaitTimeout, + }, api.runner) } -func (api runnerImportMediaAPI) Query(ctx context.Context, pippitAssetID string) error { - _, err := canvascore.Get(ctx, canvascore.GetOptions{AssetIDs: []string{pippitAssetID}}, api.runner) - return err +func (api runnerImportMediaAPI) Query(ctx context.Context, pippitAssetID string) (bool, error) { + result, err := canvascore.GetExisting(ctx, []string{pippitAssetID}, api.runner) + if err != nil { + return false, err + } + return result != nil && len(result.Assets) == 1, nil } // PreflightUpload mirrors the current Access Key authorizer's local guard. It @@ -66,12 +90,13 @@ func (api runnerImportMediaAPI) PreflightUpload(ctx context.Context) error { } type validatedImportMedia struct { - LogicalID string - MediaType string - FileName string - LocalPath string - SHA256 string - ByteSize int64 + LogicalID string + MediaType string + FileName string + LocalPath string + SHA256 string + ContentFingerprint string + ByteSize int64 } type mediaResolutionOptions struct { @@ -82,6 +107,8 @@ type mediaResolutionOptions struct { BundleRoot string CanvasJournalPath string CheckpointPath string + PollInterval time.Duration + WaitTimeout time.Duration } type mediaCheckpoint struct { @@ -93,13 +120,15 @@ type mediaCheckpoint struct { } type mediaCheckpointEntry struct { - LogicalID string `json:"logical_id"` - MediaType string `json:"media_type"` - SHA256 string `json:"sha256"` - Status string `json:"status"` - AssetID string `json:"asset_id,omitempty"` - PippitAssetID string `json:"pippit_asset_id,omitempty"` - LastError string `json:"last_error,omitempty"` + LogicalID string `json:"logical_id"` + MediaType string `json:"media_type"` + SHA256 string `json:"sha256"` + ContentFingerprint string `json:"content_fingerprint,omitempty"` + CanonicalByteSize int64 `json:"canonical_byte_size,omitempty"` + Status string `json:"status"` + AssetID string `json:"asset_id,omitempty"` + PippitAssetID string `json:"pippit_asset_id,omitempty"` + LastError string `json:"last_error,omitempty"` } func readAndValidateExportMedia(bundleDir string, plan canvasplan.Plan) ([]validatedImportMedia, error) { @@ -112,45 +141,29 @@ func readAndValidateExportMedia(bundleDir string, plan canvasplan.Plan) ([]valid if err := requireFileWithinBundle(localPath, bundleDir); err != nil { return nil, fmt.Errorf("invalid LibTV media %q: %w", requirement.LogicalID, err) } - info, err := os.Stat(localPath) + identity, err := inspectImportMediaFile(localPath) if err != nil { return nil, fmt.Errorf("inspect LibTV media %q: %w", requirement.LogicalID, err) } - if requirement.Metadata.ByteSize == nil || info.Size() != *requirement.Metadata.ByteSize { + if requirement.Metadata.ByteSize == nil || identity.ByteSize != *requirement.Metadata.ByteSize { return nil, fmt.Errorf("LibTV media %q byte size does not match CanvasPlan", requirement.LogicalID) } - digest, err := fileSHA256(localPath) - if err != nil { - return nil, fmt.Errorf("hash LibTV media %q: %w", requirement.LogicalID, err) - } - if digest != requirement.SHA256 { + if identity.RawSHA256 != requirement.SHA256 { return nil, fmt.Errorf("LibTV media %q SHA-256 does not match CanvasPlan", requirement.LogicalID) } result = append(result, validatedImportMedia{ - LogicalID: requirement.LogicalID, - MediaType: requirement.MediaType, - FileName: requirement.FileName, - LocalPath: localPath, - SHA256: digest, - ByteSize: info.Size(), + LogicalID: requirement.LogicalID, + MediaType: requirement.MediaType, + FileName: requirement.FileName, + LocalPath: localPath, + SHA256: identity.RawSHA256, + ContentFingerprint: identity.ContentFingerprint, + ByteSize: identity.ByteSize, }) } return result, nil } -func fileSHA256(path string) (string, error) { - file, err := os.Open(path) - if err != nil { - return "", err - } - defer file.Close() - hash := sha256.New() - if _, err := io.Copy(hash, file); err != nil { - return "", err - } - return hex.EncodeToString(hash.Sum(nil)), nil -} - func resolveImportMedia( ctx context.Context, opts mediaResolutionOptions, @@ -177,10 +190,15 @@ func resolveImportMedia( return canvasplan.ResolvedMediaSet{}, err } } - entries, err := validateCheckpointEntries(checkpoint, opts.Media) + entries, migrated, err := validateCheckpointEntries(checkpoint, opts) if err != nil { return canvasplan.ResolvedMediaSet{}, err } + if migrated { + if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf("save migrated canvas import media checkpoint: %w", err) + } + } if len(opts.Media) == 0 { reportImportMediaProgress(stderr, 0, 0, "complete", validatedImportMedia{FileName: "(none)"}) } @@ -210,10 +228,9 @@ func resolveImportMedia( media.LogicalID, opts.CheckpointPath, ) case mediaStatusProcessing: - reportImportMediaProgress(stderr, index, len(opts.Media), "checking", media) - if err := api.Query(ctx, existing.PippitAssetID); err != nil { + if err := waitForImportMediaReady(ctx, opts, api, stderr, index, media, existing.PippitAssetID); err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "previous media upload %q is not queryable yet; durable IDs remain in %s: %w", + "wait for previous media upload %q; durable IDs remain in %s: %w", media.LogicalID, opts.CheckpointPath, err, ) } @@ -227,12 +244,19 @@ func resolveImportMedia( case mediaStatusReady: if _, queried := queriedReadyAssetIDs[existing.PippitAssetID]; !queried { reportImportMediaProgress(stderr, index, len(opts.Media), "checking", media) - if err := api.Query(ctx, existing.PippitAssetID); err != nil { + ready, err := api.Query(ctx, existing.PippitAssetID) + if err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( "previously uploaded media %q is unavailable to the current Pippit account; refusing checkpoint reuse: %w", media.LogicalID, err, ) } + if !ready { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "previously uploaded media %q is not visible to the current Pippit account; refusing checkpoint reuse", + media.LogicalID, + ) + } queriedReadyAssetIDs[existing.PippitAssetID] = struct{}{} } default: @@ -244,21 +268,30 @@ func resolveImportMedia( if duplicate := readyEntryByDigest(entries, media); duplicate != nil { if _, queried := queriedReadyAssetIDs[duplicate.PippitAssetID]; !queried { reportImportMediaProgress(stderr, index, len(opts.Media), "checking", media) - if err := api.Query(ctx, duplicate.PippitAssetID); err != nil { + ready, err := api.Query(ctx, duplicate.PippitAssetID) + if err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( "deduplicated media %q is unavailable to the current Pippit account; refusing checkpoint reuse: %w", media.LogicalID, err, ) } + if !ready { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "deduplicated media %q is not visible to the current Pippit account; refusing checkpoint reuse", + media.LogicalID, + ) + } queriedReadyAssetIDs[duplicate.PippitAssetID] = struct{}{} } entry := mediaCheckpointEntry{ - LogicalID: media.LogicalID, - MediaType: media.MediaType, - SHA256: media.SHA256, - Status: mediaStatusReady, - AssetID: duplicate.AssetID, - PippitAssetID: duplicate.PippitAssetID, + LogicalID: media.LogicalID, + MediaType: media.MediaType, + SHA256: media.SHA256, + ContentFingerprint: media.ContentFingerprint, + CanonicalByteSize: media.ByteSize, + Status: mediaStatusReady, + AssetID: duplicate.AssetID, + PippitAssetID: duplicate.PippitAssetID, } entries[media.LogicalID] = &entry if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { @@ -276,11 +309,13 @@ func resolveImportMedia( } } entry := mediaCheckpointEntry{ - LogicalID: media.LogicalID, - MediaType: media.MediaType, - SHA256: media.SHA256, - Status: mediaStatusUploadRequested, - LastError: "upload request is about to be dispatched; interruption requires manual outcome confirmation", + LogicalID: media.LogicalID, + MediaType: media.MediaType, + SHA256: media.SHA256, + ContentFingerprint: media.ContentFingerprint, + CanonicalByteSize: media.ByteSize, + Status: mediaStatusUploadRequested, + LastError: "upload request is about to be dispatched; interruption requires manual outcome confirmation", } if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( @@ -290,7 +325,7 @@ func resolveImportMedia( } entries[media.LogicalID] = &entry reportImportMediaProgress(stderr, index, len(opts.Media), "uploading", media) - uploaded, uploadErr := api.Upload(ctx, media.LocalPath) + uploaded, uploadErr := api.Upload(ctx, media) if uploadErr != nil && strings.Contains(uploadErr.Error(), "XYQ_ACCESS_KEY 缺失") { if checkpointErr := removeAndSaveMediaEntry(opts.CheckpointPath, checkpoint, media.LogicalID); checkpointErr != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( @@ -343,13 +378,15 @@ func resolveImportMedia( if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { return canvasplan.ResolvedMediaSet{}, err } - reportImportMediaProgress(stderr, index+1, len(opts.Media), "uploaded", media) - return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "media upload %q is still processing; durable IDs are checkpointed in %s, rerun to query without re-uploading", - media.LogicalID, opts.CheckpointPath, - ) + if err := waitForImportMediaReady(ctx, opts, api, stderr, index, media, entry.PippitAssetID); err != nil { + return canvasplan.ResolvedMediaSet{}, fmt.Errorf( + "wait for media upload %q; durable IDs remain in %s: %w", + media.LogicalID, opts.CheckpointPath, err, + ) + } } entry.Status = mediaStatusReady + entry.LastError = "" queriedReadyAssetIDs[entry.PippitAssetID] = struct{}{} entries[media.LogicalID] = &entry if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { @@ -375,6 +412,62 @@ func resolveImportMedia( return resolved, nil } +func waitForImportMediaReady( + ctx context.Context, + opts mediaResolutionOptions, + api importMediaAPI, + stderr io.Writer, + processed int, + media validatedImportMedia, + pippitAssetID string, +) error { + pollInterval := opts.PollInterval + if pollInterval <= 0 { + pollInterval = defaultImportMediaPollInterval + } + waitTimeout := opts.WaitTimeout + if waitTimeout <= 0 { + waitTimeout = defaultImportMediaWaitTimeout + } + waitCtx, cancel := context.WithTimeout(ctx, waitTimeout) + defer cancel() + for attempt := 1; ; attempt++ { + action := "waiting" + if attempt == 1 { + action = "processing" + } + reportImportMediaProgress(stderr, processed, len(opts.Media), action, media) + ready, err := api.Query(waitCtx, pippitAssetID) + if err != nil { + if waitCtx.Err() != nil { + return importMediaWaitError(waitCtx.Err(), pippitAssetID, waitTimeout) + } + return fmt.Errorf( + "query processing Pippit asset %q failed; this is a read/authentication error, not a processing signal: %w", + pippitAssetID, + err, + ) + } + if ready { + return nil + } + timer := time.NewTimer(pollInterval) + select { + case <-waitCtx.Done(): + timer.Stop() + return importMediaWaitError(waitCtx.Err(), pippitAssetID, waitTimeout) + case <-timer.C: + } + } +} + +func importMediaWaitError(err error, pippitAssetID string, timeout time.Duration) error { + if err == context.DeadlineExceeded { + return fmt.Errorf("Pippit asset %q was not visible within %s; it will only be queried on the next run and will not be uploaded again", pippitAssetID, timeout) + } + return fmt.Errorf("wait for Pippit asset %q canceled; it will not be uploaded again: %w", pippitAssetID, err) +} + func reportImportMediaProgress( stderr io.Writer, processed int, @@ -454,29 +547,133 @@ func loadMediaCheckpoint(opts mediaResolutionOptions) (*mediaCheckpoint, error) func validateCheckpointEntries( checkpoint *mediaCheckpoint, - media []validatedImportMedia, -) (map[string]*mediaCheckpointEntry, error) { - expected := make(map[string]validatedImportMedia, len(media)) - for _, item := range media { + opts mediaResolutionOptions, +) (map[string]*mediaCheckpointEntry, bool, error) { + expected := make(map[string]validatedImportMedia, len(opts.Media)) + for _, item := range opts.Media { expected[item.LogicalID] = item } entries := make(map[string]*mediaCheckpointEntry, len(checkpoint.Entries)) + migrated := false for index := range checkpoint.Entries { entry := &checkpoint.Entries[index] item, ok := expected[entry.LogicalID] - if !ok || item.MediaType != entry.MediaType || item.SHA256 != entry.SHA256 { - return nil, fmt.Errorf("canvas import media changed after checkpoint creation") + if !ok || item.MediaType != entry.MediaType { + return nil, false, fmt.Errorf("canvas import media changed after checkpoint creation") + } + if !validRawMediaSHA256(entry.SHA256) || !validImportMediaContentFingerprint(item.ContentFingerprint) { + return nil, false, fmt.Errorf("canvas import media checkpoint entry %q has an invalid fingerprint", entry.LogicalID) + } + if item.SHA256 != entry.SHA256 { + if item.MediaType != "image" { + return nil, false, fmt.Errorf("canvas import media changed after checkpoint creation") + } + if entry.ContentFingerprint == "" || entry.CanonicalByteSize == 0 { + previousIdentity, err := findLegacyCheckpointImageIdentity(checkpoint, opts, *entry) + if err != nil { + return nil, false, fmt.Errorf("verify changed checkpoint image %q: %w", entry.LogicalID, err) + } + if !validImportMediaContentFingerprint(previousIdentity.ContentFingerprint) { + return nil, false, fmt.Errorf("canvas import media checkpoint image %q has an invalid content fingerprint", entry.LogicalID) + } + if entry.ContentFingerprint != "" && entry.ContentFingerprint != previousIdentity.ContentFingerprint { + return nil, false, fmt.Errorf("canvas import media checkpoint image %q content fingerprint does not match its retained bundle", entry.LogicalID) + } + if entry.CanonicalByteSize != 0 && entry.CanonicalByteSize != previousIdentity.ByteSize { + return nil, false, fmt.Errorf("canvas import media checkpoint image %q byte size does not match its retained bundle", entry.LogicalID) + } + if entry.ContentFingerprint == "" { + entry.ContentFingerprint = previousIdentity.ContentFingerprint + migrated = true + } + if entry.CanonicalByteSize == 0 { + entry.CanonicalByteSize = previousIdentity.ByteSize + migrated = true + } + } + if !validImportMediaContentFingerprint(entry.ContentFingerprint) || entry.ContentFingerprint != item.ContentFingerprint { + return nil, false, fmt.Errorf("canvas import image content changed after checkpoint creation") + } + } else { + if entry.ContentFingerprint != "" && + (!validImportMediaContentFingerprint(entry.ContentFingerprint) || entry.ContentFingerprint != item.ContentFingerprint) { + return nil, false, fmt.Errorf("canvas import media content changed after checkpoint creation") + } + if entry.ContentFingerprint == "" { + entry.ContentFingerprint = item.ContentFingerprint + migrated = true + } + if entry.CanonicalByteSize != 0 && entry.CanonicalByteSize != item.ByteSize { + return nil, false, fmt.Errorf("canvas import media byte size changed after checkpoint creation") + } + if entry.CanonicalByteSize == 0 { + entry.CanonicalByteSize = item.ByteSize + migrated = true + } } if _, duplicate := entries[entry.LogicalID]; duplicate { - return nil, fmt.Errorf("canvas import media checkpoint contains duplicate logical ID %q", entry.LogicalID) + return nil, false, fmt.Errorf("canvas import media checkpoint contains duplicate logical ID %q", entry.LogicalID) } if (entry.Status == mediaStatusReady || entry.Status == mediaStatusProcessing) && (strings.TrimSpace(entry.AssetID) == "" || strings.TrimSpace(entry.PippitAssetID) == "") { - return nil, fmt.Errorf("canvas import media checkpoint entry %q has no durable IDs", entry.LogicalID) + return nil, false, fmt.Errorf("canvas import media checkpoint entry %q has no durable IDs", entry.LogicalID) + } + entries[entry.LogicalID] = entry + } + return entries, migrated, nil +} + +func validRawMediaSHA256(value string) bool { + if len(value) != sha256.Size*2 || strings.ToLower(value) != value { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +func canonicalizeImportPlanMedia( + plan canvasplan.Plan, + target string, + journalPath string, + checkpointPath string, +) (canvasplan.Plan, error) { + lock, err := acquireImportMediaCheckpointLock(checkpointPath + ".lock") + if err != nil { + return canvasplan.Plan{}, err + } + defer func() { _ = lock.release() }() + checkpoint, err := loadMediaCheckpoint(mediaResolutionOptions{ + Plan: plan, + Target: target, + CanvasJournalPath: journalPath, + CheckpointPath: checkpointPath, + }) + if err != nil { + return canvasplan.Plan{}, fmt.Errorf("load canonical canvas import media identities: %w", err) + } + entries := make(map[string]mediaCheckpointEntry, len(checkpoint.Entries)) + for _, entry := range checkpoint.Entries { + if _, exists := entries[entry.LogicalID]; exists { + return canvasplan.Plan{}, fmt.Errorf("canvas import media checkpoint contains duplicate logical ID %q", entry.LogicalID) } entries[entry.LogicalID] = entry } - return entries, nil + canonical := plan + canonical.RequiredMedia = append([]canvasplan.MediaRequirement(nil), plan.RequiredMedia...) + for index := range canonical.RequiredMedia { + requirement := &canonical.RequiredMedia[index] + entry, ok := entries[requirement.LogicalID] + if !ok || entry.MediaType != requirement.MediaType || entry.Status != mediaStatusReady { + return canvasplan.Plan{}, fmt.Errorf("canvas import media checkpoint has no ready canonical identity for %q", requirement.LogicalID) + } + if !validRawMediaSHA256(entry.SHA256) || entry.CanonicalByteSize <= 0 { + return canvasplan.Plan{}, fmt.Errorf("canvas import media checkpoint has an invalid canonical identity for %q", requirement.LogicalID) + } + requirement.SHA256 = entry.SHA256 + byteSize := entry.CanonicalByteSize + requirement.Metadata.ByteSize = &byteSize + } + return canonical, nil } func readyEntryByDigest(entries map[string]*mediaCheckpointEntry, media validatedImportMedia) *mediaCheckpointEntry { From 10f2b65abd5547f6ed1b34a692ec759afd5434c5 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 19:09:21 +0800 Subject: [PATCH 24/48] fix(canvas): reconcile ambiguous canvas writes Co-authored-by: Codex <codex@openai.com> --- internal/canvasplan/executor.go | 141 ++++++++---- internal/canvasplan/reconcile.go | 381 +++++++++++++++++++++++++++++++ internal/canvasplan/types.go | 1 + 3 files changed, 480 insertions(+), 43 deletions(-) create mode 100644 internal/canvasplan/reconcile.go diff --git a/internal/canvasplan/executor.go b/internal/canvasplan/executor.go index ba04877..3ea7e82 100644 --- a/internal/canvasplan/executor.go +++ b/internal/canvasplan/executor.go @@ -151,38 +151,36 @@ func (executor *Executor) reverifyCompleted( assetIDs := DocumentAssetIDs(document) queried, err := executor.api.Get(ctx, assetIDs) if err != nil { - journal.Verification = &Verification{ExpectedAssetCount: len(assetIDs), Verified: false} - return failExecution( - journalPath, - journal, - plan, - StateVerificationFailed, - fmt.Errorf("verify completed CanvasPlan with current authentication: %w", err), - ) + result := executionResult(journalPath, journal, plan) + result.Warning = fmt.Sprintf("query previously verified Canvas with current authentication: %v", err) + return result, fmt.Errorf("%s", result.Warning) } - verification := VerifyDocument(document, queried.Assets) + if queried == nil { + result := executionResult(journalPath, journal, plan) + result.Warning = "query previously verified Canvas with current authentication returned no result" + return result, fmt.Errorf("%s", result.Warning) + } + verification := verifyJournalAssetHashes(journal.AssetSHA256, queried.Assets) verification.LogID = queried.LogID - journal.Verification = &verification - if !verification.Verified { - return failExecution( - journalPath, - journal, - plan, - StateVerificationFailed, - fmt.Errorf( - "completed CanvasPlan no longer matches current authenticated assets: missing=%d unverifiable=%d mismatched=%d", - len(verification.MissingAssetIDs), - len(verification.UnverifiableAssetIDs), - len(verification.MismatchedAssetIDs), - ), + verification.RecoveredFromQuery = true + if len(verification.MissingAssetIDs) != 0 || len(verification.UnverifiableAssetIDs) != 0 { + result := executionResult(journalPath, journal, plan) + result.Warning = fmt.Sprintf( + "previously verified Canvas is not fully accessible with current authentication: missing=%d unverifiable=%d", + len(verification.MissingAssetIDs), + len(verification.UnverifiableAssetIDs), + ) + return result, fmt.Errorf("%s", result.Warning) + } + changed := append([]string(nil), verification.MismatchedAssetIDs...) + result := executionResult(journalPath, journal, plan) + if len(changed) != 0 { + result.Warning = fmt.Sprintf( + "%d Canvas asset(s) changed after the import was originally verified; current access is valid and apply was not replayed", + len(changed), ) } - journal.State = StateVerified - journal.LastError = "" - if err := saveJournal(journalPath, journal); err != nil { - return executionResult(journalPath, journal, plan), err - } - return executionResult(journalPath, journal, plan), nil + return result, nil } func (executor *Executor) ensureRoot( @@ -388,16 +386,11 @@ func (executor *Executor) applyAndVerify( } applyResult, err := executor.api.Apply(ctx, canvas.ApplyOptions{ProjectID: journal.Create.ProjectID, Request: request}) if err != nil { - journal.Apply.Status = "ambiguous" - return failExecution(journalPath, journal, plan, StateApplyAmbiguous, err) + return executor.recoverAmbiguousApplyByQuery(ctx, journalPath, journal, plan, document, assetIDs, err) } if applyResult == nil || len(applyResult.Results) != 1 { - journal.Apply.Status = "ambiguous" - return failExecution( - journalPath, - journal, - plan, - StateApplyAmbiguous, + return executor.recoverAmbiguousApplyByQuery( + ctx, journalPath, journal, plan, document, assetIDs, fmt.Errorf("canvas apply acknowledgement is incomplete; query affected assets and do not replay blindly"), ) } @@ -437,6 +430,67 @@ func (executor *Executor) applyAndVerify( return executionResult(journalPath, journal, plan), nil } +func (executor *Executor) recoverAmbiguousApplyByQuery( + ctx context.Context, + journalPath string, + journal *Journal, + plan Plan, + document *Document, + assetIDs []string, + applyCause error, +) (*ExecutionResult, error) { + journal.Apply.Status = "ambiguous" + journal.State = StateApplyAmbiguous + journal.LastError = sanitizeJournalError(applyCause.Error()) + if err := saveJournal(journalPath, journal); err != nil { + return executionResult(journalPath, journal, plan), fmt.Errorf( + "%w; additionally failed to persist ambiguous Canvas apply: %v", + applyCause, + err, + ) + } + + queried, queryErr := executor.api.Get(ctx, assetIDs) + if queryErr != nil { + return executionResult(journalPath, journal, plan), fmt.Errorf( + "%w; exact query-back after the ambiguous response also failed: %v", + applyCause, + queryErr, + ) + } + if queried == nil { + return executionResult(journalPath, journal, plan), fmt.Errorf( + "%w; exact query-back after the ambiguous response returned no result", + applyCause, + ) + } + verification := VerifyDocument(document, queried.Assets) + verification.LogID = queried.LogID + verification.RecoveredFromQuery = true + if !verification.Verified { + result := executionResult(journalPath, journal, plan) + result.Verification = &verification + return result, fmt.Errorf( + "%w; exact query-back after the ambiguous response did not verify: missing=%d unverifiable=%d mismatched=%d; apply was not replayed", + applyCause, + len(verification.MissingAssetIDs), + len(verification.UnverifiableAssetIDs), + len(verification.MismatchedAssetIDs), + ) + } + + journal.Verification = &verification + journal.Apply.Status = "verified" + journal.State = StateVerified + journal.LastError = "" + if err := saveJournal(journalPath, journal); err != nil { + return executionResult(journalPath, journal, plan), err + } + result := executionResult(journalPath, journal, plan) + result.Warning = "Canvas apply response was ambiguous; all expected assets were recovered by exact query-back and apply was not replayed" + return result, nil +} + func prepareApplyRequest(journal *Journal, document *Document, rootVersion int64) (canvas.ApplyRequest, error) { if journal.Create == nil || !isPositiveDecimal(journal.Create.ProjectID) { return canvas.ApplyRequest{}, fmt.Errorf("created personal novel project_id must be a positive decimal JSON string") @@ -492,13 +546,14 @@ func executionResult(journalPath string, journal *Journal, plan Plan) *Execution return nil } result := &ExecutionResult{ - State: journal.State, - JournalPath: journalPath, - OperationID: journal.OperationID, - DocumentSHA256: journal.DocumentSHA256, - NodeCount: len(plan.Nodes) + len(plan.Groups), - EdgeCount: len(plan.Edges), - Verification: journal.Verification, + State: journal.State, + JournalPath: journalPath, + OperationID: journal.OperationID, + DocumentSHA256: journal.DocumentSHA256, + NodeCount: len(plan.Nodes) + len(plan.Groups), + EdgeCount: len(plan.Edges), + DegradationCount: len(plan.Degradations), + Verification: journal.Verification, } if journal.DocumentSHA256 != "" { result.AssetCount = len(journal.AssetSHA256) diff --git a/internal/canvasplan/reconcile.go b/internal/canvasplan/reconcile.go new file mode 100644 index 0000000..01769af --- /dev/null +++ b/internal/canvasplan/reconcile.go @@ -0,0 +1,381 @@ +package canvasplan + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +// ErrReconcileNotEligible indicates that normal Execute/Resume state handling +// must continue; no remote write was issued by Reconcile. +var ErrReconcileNotEligible = errors.New("CanvasPlan journal is not eligible for journal-only reconciliation") + +// Reconcile verifies an existing execution journal from its persisted asset +// hashes. It never creates, allocates, or applies Canvas assets. +func Reconcile(ctx context.Context, journalPath string, runner *common.Runner) (*ExecutionResult, error) { + if runner == nil || runner.Client == nil { + return nil, fmt.Errorf("CanvasPlan runner client is missing") + } + return NewExecutor(runner).Reconcile(ctx, journalPath) +} + +// ReconcileWithInputs verifies a journal only after binding it to the current +// normalized plan and resolved-media inputs. Import orchestrators should use +// this form so an explicit journal cannot be resumed for a different source. +func ReconcileWithInputs( + ctx context.Context, + journalPath string, + plan Plan, + resolved ResolvedMediaSet, + runner *common.Runner, +) (*ExecutionResult, error) { + if runner == nil || runner.Client == nil { + return nil, fmt.Errorf("CanvasPlan runner client is missing") + } + return NewExecutor(runner).ReconcileWithInputs(ctx, journalPath, plan, resolved) +} + +// Reconcile verifies an ambiguous or previously verified journal without the +// original plan or resolved-media inputs and without replaying Apply. +func (executor *Executor) Reconcile( + ctx context.Context, + journalPath string, +) (result *ExecutionResult, returnErr error) { + return executor.reconcile(ctx, journalPath, "", "", nil) +} + +// ReconcileWithInputs binds reconciliation to the exact normalized plan and +// resolved-media identity already recorded by the execution journal. +func (executor *Executor) ReconcileWithInputs( + ctx context.Context, + journalPath string, + inputPlan Plan, + inputResolved ResolvedMediaSet, +) (*ExecutionResult, error) { + plan, err := NormalizePlan(inputPlan) + if err != nil { + return nil, err + } + resolved, err := NormalizeResolvedMedia(inputResolved) + if err != nil { + return nil, err + } + if err := ValidateResolution(plan, resolved); err != nil { + return nil, err + } + planHash, err := hashJSON(plan) + if err != nil { + return nil, fmt.Errorf("hash CanvasPlan for reconciliation: %w", err) + } + resolvedHash, err := hashJSON(resolved) + if err != nil { + return nil, fmt.Errorf("hash resolved media for reconciliation: %w", err) + } + return executor.reconcile(ctx, journalPath, planHash, resolvedHash, &plan) +} + +func (executor *Executor) reconcile( + ctx context.Context, + journalPath string, + expectedPlanHash string, + expectedResolvedHash string, + plan *Plan, +) (result *ExecutionResult, returnErr error) { + if executor == nil || executor.api == nil { + return nil, fmt.Errorf("CanvasPlan executor API is missing") + } + journalPath = strings.TrimSpace(journalPath) + if journalPath == "" { + return nil, fmt.Errorf("CanvasPlan journal path is required") + } + absolute, err := filepath.Abs(journalPath) + if err != nil { + return nil, fmt.Errorf("resolve CanvasPlan journal path: %w", err) + } + absolute = filepath.Clean(absolute) + + lock, err := acquireJournalLock(absolute) + if err != nil { + return nil, err + } + defer func() { + if releaseErr := lock.release(); releaseErr != nil { + if returnErr == nil { + returnErr = fmt.Errorf("release CanvasPlan journal lock: %w", releaseErr) + } else { + returnErr = fmt.Errorf("%w; release CanvasPlan journal lock: %v", returnErr, releaseErr) + } + } + }() + + journal, err := loadReconcileJournal(absolute) + if err != nil { + return nil, err + } + result = reconciliationResult(absolute, journal, plan) + if err := validateReconcileJournal(journal); err != nil { + return result, err + } + if expectedPlanHash != "" && + (journal.PlanSHA256 != expectedPlanHash || journal.ResolvedMediaSHA256 != expectedResolvedHash) { + return result, fmt.Errorf("CanvasPlan or resolved media changed after journal creation") + } + + assetIDs := make([]string, 0, len(journal.AssetSHA256)) + for assetID := range journal.AssetSHA256 { + assetIDs = append(assetIDs, assetID) + } + sort.Strings(assetIDs) + wasVerified := journalWasVerified(journal) + queried, err := executor.api.Get(ctx, assetIDs) + if err != nil { + result.Warning = fmt.Sprintf("query CanvasPlan assets with current authentication: %v", err) + return result, fmt.Errorf("%s", result.Warning) + } + if queried == nil { + result.Warning = "query CanvasPlan assets with current authentication returned no result" + return result, fmt.Errorf("%s", result.Warning) + } + + verification := verifyJournalAssetHashes(journal.AssetSHA256, queried.Assets) + verification.LogID = queried.LogID + verification.RecoveredFromQuery = true + if wasVerified { + if len(verification.MissingAssetIDs) != 0 || len(verification.UnverifiableAssetIDs) != 0 { + result.Warning = fmt.Sprintf( + "previously verified Canvas is not fully accessible with current authentication: missing=%d unverifiable=%d", + len(verification.MissingAssetIDs), + len(verification.UnverifiableAssetIDs), + ) + return result, fmt.Errorf("%s", result.Warning) + } + changed := append([]string(nil), verification.MismatchedAssetIDs...) + if journal.State == StateVerified { + result = reconciliationResult(absolute, journal, plan) + if len(changed) != 0 { + result.Warning = fmt.Sprintf( + "%d Canvas asset(s) changed after the import was originally verified; current access is valid and apply was not replayed", + len(changed), + ) + } + return result, nil + } + // This state is only used to recover a completed import that was + // incorrectly demoted by a later current-account query. Keep the + // original strict verification as the audit record; the current query + // establishes accessibility, not content immutability. + if journal.Verification == nil || !journal.Verification.Verified { + // Older clients replaced the original strict verification with a + // later failed current-account check. Apply.Status="verified" is only + // persisted after strict query-back succeeds, so rebuild the lost + // completion marker without treating current content edits as drift. + journal.Verification = &Verification{ + ExpectedAssetCount: len(assetIDs), + ReturnedAssetCount: len(queried.Assets), + Verified: true, + RecoveredFromQuery: true, + LogID: queried.LogID, + } + } + journal.State = StateVerified + journal.LastError = "" + if err := saveJournal(absolute, journal); err != nil { + return reconciliationResult(absolute, journal, plan), err + } + result = reconciliationResult(absolute, journal, plan) + if len(changed) != 0 { + result.Warning = fmt.Sprintf( + "%d Canvas asset(s) changed after the import was originally verified; current access is valid and apply was not replayed", + len(changed), + ) + } + return result, nil + } + if !verification.Verified { + result.Verification = &verification + result.Warning = fmt.Sprintf( + "CanvasPlan journal reconciliation failed: missing=%d unverifiable=%d mismatched=%d; apply was not replayed", + len(verification.MissingAssetIDs), + len(verification.UnverifiableAssetIDs), + len(verification.MismatchedAssetIDs), + ) + return result, fmt.Errorf("%s", result.Warning) + } + + journal.Verification = &verification + journal.State = StateVerified + journal.LastError = "" + if journal.Apply != nil { + journal.Apply.Status = "verified" + } + if err := saveJournal(absolute, journal); err != nil { + return reconciliationResult(absolute, journal, plan), err + } + return reconciliationResult(absolute, journal, plan), nil +} + +func journalWasVerified(journal *Journal) bool { + if journal == nil { + return false + } + if journal.State == StateVerified { + return journal.Verification != nil && journal.Verification.Verified + } + return journal.State == StateVerificationFailed && journal.Apply != nil && journal.Apply.Status == "verified" +} + +func loadReconcileJournal(path string) (*Journal, error) { + directory, err := ensureSecureJournalDirectory(filepath.Dir(path)) + if err != nil { + return nil, err + } + file, err := openRegularFileNoFollow(path, os.O_RDWR, 0) + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("CanvasPlan journal does not exist: %s", path) + } + return nil, fmt.Errorf("open CanvasPlan journal: %w", err) + } + defer file.Close() + if err := file.Chmod(0o600); err != nil { + return nil, fmt.Errorf("secure CanvasPlan journal permissions: %w", err) + } + journal, err := decodeJournal(io.LimitReader(file, maxJournalBytes+1)) + if err != nil { + return nil, err + } + if journal.Schema != JournalSchema { + return nil, fmt.Errorf("unsupported CanvasPlan journal schema %q", journal.Schema) + } + if strings.TrimSpace(journal.OperationID) == "" || strings.TrimSpace(journal.RequestID) == "" || strings.TrimSpace(journal.State) == "" { + return nil, fmt.Errorf("CanvasPlan journal identity or state is incomplete") + } + if err := directory.validateStable(); err != nil { + return nil, err + } + return journal, nil +} + +func validateReconcileJournal(journal *Journal) error { + if !sha256Pattern.MatchString(journal.PlanSHA256) || !sha256Pattern.MatchString(journal.ResolvedMediaSHA256) { + return fmt.Errorf("CanvasPlan journal has invalid plan or resolved-media SHA-256") + } + switch journal.State { + case StateApplyAmbiguous: + if journal.Apply == nil || journal.Apply.Status != "ambiguous" { + return fmt.Errorf("CanvasPlan apply-ambiguous journal has no ambiguous apply record") + } + case StateVerified: + if journal.Verification == nil || !journal.Verification.Verified { + return fmt.Errorf("verified CanvasPlan journal is missing successful historical verification") + } + case StateVerificationFailed: + if journal.Apply == nil || journal.Apply.Status != "verified" { + return fmt.Errorf("%w: state %q has no prior verified apply", ErrReconcileNotEligible, journal.State) + } + default: + return fmt.Errorf("%w: state %q", ErrReconcileNotEligible, journal.State) + } + if journal.Create == nil { + return fmt.Errorf("CanvasPlan journal is missing its create result") + } + if !sha256Pattern.MatchString(journal.DocumentSHA256) { + return fmt.Errorf("CanvasPlan journal has no valid document SHA-256") + } + if len(journal.AssetSHA256) == 0 { + return fmt.Errorf("CanvasPlan journal has no asset SHA-256 entries") + } + for assetID, digest := range journal.AssetSHA256 { + if strings.TrimSpace(assetID) == "" || assetID != strings.TrimSpace(assetID) || !sha256Pattern.MatchString(digest) { + return fmt.Errorf("CanvasPlan journal contains an invalid asset SHA-256 entry") + } + } + return nil +} + +func verifyJournalAssetHashes(expected map[string]string, assets []json.RawMessage) Verification { + verification := Verification{ExpectedAssetCount: len(expected), ReturnedAssetCount: len(assets)} + seen := make(map[string]struct{}, len(assets)) + for index, asset := range assets { + assetID, err := queriedAssetID(asset) + if err != nil { + verification.UnverifiableAssetIDs = append(verification.UnverifiableAssetIDs, fmt.Sprintf("response[%d]", index)) + continue + } + expectedHash, requested := expected[assetID] + if !requested { + verification.UnverifiableAssetIDs = append(verification.UnverifiableAssetIDs, assetID) + continue + } + if _, duplicate := seen[assetID]; duplicate { + verification.UnverifiableAssetIDs = append(verification.UnverifiableAssetIDs, assetID) + continue + } + seen[assetID] = struct{}{} + content, err := queriedAssetContent(asset) + if err != nil { + verification.UnverifiableAssetIDs = append(verification.UnverifiableAssetIDs, assetID) + continue + } + storedHash, err := hashRawJSON(content) + if err != nil { + verification.UnverifiableAssetIDs = append(verification.UnverifiableAssetIDs, assetID) + continue + } + if storedHash != expectedHash { + verification.MismatchedAssetIDs = append(verification.MismatchedAssetIDs, assetID) + } + } + for assetID := range expected { + if _, exists := seen[assetID]; !exists { + verification.MissingAssetIDs = append(verification.MissingAssetIDs, assetID) + } + } + sort.Strings(verification.MissingAssetIDs) + sort.Strings(verification.UnverifiableAssetIDs) + sort.Strings(verification.MismatchedAssetIDs) + verification.Verified = len(verification.MissingAssetIDs) == 0 && + len(verification.UnverifiableAssetIDs) == 0 && + len(verification.MismatchedAssetIDs) == 0 + return verification +} + +func reconciliationResult(journalPath string, journal *Journal, plan *Plan) *ExecutionResult { + if journal == nil { + return nil + } + result := &ExecutionResult{ + State: journal.State, + JournalPath: journalPath, + OperationID: journal.OperationID, + DocumentSHA256: journal.DocumentSHA256, + AssetCount: len(journal.AssetSHA256), + Verification: journal.Verification, + } + if plan != nil { + result.NodeCount = len(plan.Nodes) + len(plan.Groups) + result.EdgeCount = len(plan.Edges) + result.DegradationCount = len(plan.Degradations) + } + if journal.Create != nil { + result.ProjectID = journal.Create.ProjectID + result.RootCanvasID = journal.Create.CanvasAssetID + result.OverviewPippitAssetID = journal.Create.OverviewPippitAssetID + result.WebURL = journal.Create.WebURL + } + if journal.Apply != nil { + result.TransactionID = journal.Apply.TransactionID + } + if journal.State != StateVerified { + result.Warning = journal.LastError + } + return result +} diff --git a/internal/canvasplan/types.go b/internal/canvasplan/types.go index b24c08d..910906d 100644 --- a/internal/canvasplan/types.go +++ b/internal/canvasplan/types.go @@ -163,6 +163,7 @@ type ExecutionResult struct { AssetCount int `json:"asset_count,omitempty"` NodeCount int `json:"node_count,omitempty"` EdgeCount int `json:"edge_count,omitempty"` + DegradationCount int `json:"degradation_count,omitempty"` TransactionID string `json:"transaction_id,omitempty"` Verification *Verification `json:"verification,omitempty"` Warning string `json:"warning,omitempty"` From f498e63d10651f6c73706bef7a1a966a92f5a62d Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 19:09:31 +0800 Subject: [PATCH 25/48] test(canvas): cover safe write reconciliation Co-authored-by: Codex <codex@openai.com> --- internal/canvasplan/canvasplan_test.go | 57 +++- internal/canvasplan/reconcile_test.go | 425 +++++++++++++++++++++++++ 2 files changed, 466 insertions(+), 16 deletions(-) create mode 100644 internal/canvasplan/reconcile_test.go diff --git a/internal/canvasplan/canvasplan_test.go b/internal/canvasplan/canvasplan_test.go index 1e142a7..1764c25 100644 --- a/internal/canvasplan/canvasplan_test.go +++ b/internal/canvasplan/canvasplan_test.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "strings" "testing" @@ -232,7 +233,7 @@ func TestExecutorAppliesOnceVerifiesAndReusesJournal(t *testing.T) { } } -func TestExecutorVerifiedResumeFailsOnCurrentAccountMismatch(t *testing.T) { +func TestExecutorVerifiedResumeRejectsCurrentAccountMismatchWithoutOverwritingHistory(t *testing.T) { plan, resolved := testPlanAndResolved() api := newFakeCanvasAPI(len(plan.Nodes)) executor := &Executor{api: api} @@ -242,20 +243,34 @@ func TestExecutorVerifiedResumeFailsOnCurrentAccountMismatch(t *testing.T) { if err != nil || result.State != StateVerified { t.Fatalf("first Execute() result=%#v error=%v, want verified", result, err) } + beforeBytes, err := os.ReadFile(journalPath) + if err != nil { + t.Fatal(err) + } api.getErr = errors.New("assets are not visible to current account") result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) - if err == nil || result == nil || result.State != StateVerificationFailed { - t.Fatalf("verified resume result=%#v error=%v, want current-auth verification failure", result, err) + if err == nil || result == nil || result.State != StateVerified { + t.Fatalf("verified resume result=%#v error=%v, want access error with historical verified state", result, err) } - if result.Verification == nil || result.Verification.Verified { - t.Fatalf("verification = %#v, want fresh failed verification", result.Verification) + if result.Verification == nil || !result.Verification.Verified || !strings.Contains(result.Warning, "current authentication") { + t.Fatalf("result = %#v, want preserved verification and actionable access warning", result) } if api.applyCalls != 1 { t.Fatalf("verified resume replayed apply: calls=%d", api.applyCalls) } + if journal := readJournal(t, journalPath); journal.State != StateVerified || journal.Verification == nil || !journal.Verification.Verified { + t.Fatalf("saved journal = %#v, want preserved verified history", journal) + } + afterBytes, readErr := os.ReadFile(journalPath) + if readErr != nil { + t.Fatal(readErr) + } + if !bytes.Equal(afterBytes, beforeBytes) { + t.Fatal("verified journal bytes changed after current-account query failure") + } } -func TestExecutorVerifiedResumeFailsOnRemoteDrift(t *testing.T) { +func TestExecutorVerifiedResumeAllowsNormalRemoteEdits(t *testing.T) { plan, resolved := testPlanAndResolved() api := newFakeCanvasAPI(len(plan.Nodes)) executor := &Executor{api: api} @@ -265,17 +280,21 @@ func TestExecutorVerifiedResumeFailsOnRemoteDrift(t *testing.T) { if err != nil || result.State != StateVerified { t.Fatalf("first Execute() result=%#v error=%v, want verified", result, err) } + historicalVerification := *result.Verification api.corruptGet = true result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) - if err == nil || result == nil || result.State != StateVerificationFailed { - t.Fatalf("verified resume result=%#v error=%v, want remote-drift failure", result, err) + if err != nil || result == nil || result.State != StateVerified { + t.Fatalf("verified resume result=%#v error=%v, want accessible edited Canvas", result, err) } - if result.Verification == nil || result.Verification.Verified || len(result.Verification.MismatchedAssetIDs) != 1 { - t.Fatalf("verification = %#v, want one mismatched asset", result.Verification) + if result.Verification == nil || !result.Verification.Verified || !strings.Contains(result.Warning, "changed after") { + t.Fatalf("result = %#v, want verified access with edit warning", result) } if api.applyCalls != 1 { t.Fatalf("verified resume replayed apply: calls=%d", api.applyCalls) } + if journal := readJournal(t, journalPath); journal.Verification == nil || !reflect.DeepEqual(*journal.Verification, historicalVerification) { + t.Fatalf("verified resume overwrote historical verification: before=%#v after=%#v", historicalVerification, journal.Verification) + } } func TestExecutorNeverReplaysAmbiguousApply(t *testing.T) { @@ -305,7 +324,7 @@ func TestExecutorNeverReplaysAmbiguousApply(t *testing.T) { } } -func TestExecutorRecoversCommittedAmbiguousApplyByQuery(t *testing.T) { +func TestExecutorRecoversCommittedAmbiguousApplyByQueryInSameInvocation(t *testing.T) { plan, resolved := testPlanAndResolved() api := newFakeCanvasAPI(len(plan.Nodes)) api.applyErr = errors.New("connection lost after commit") @@ -314,12 +333,11 @@ func TestExecutorRecoversCommittedAmbiguousApplyByQuery(t *testing.T) { journalPath := filepath.Join(t.TempDir(), "committed-ambiguous.json") result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) - if err == nil || result.State != StateApplyAmbiguous { - t.Fatalf("first Execute() result=%#v error=%v, want ambiguity", result, err) - } - result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) if err != nil || result.State != StateVerified || result.Verification == nil || !result.Verification.RecoveredFromQuery { - t.Fatalf("resume result=%#v error=%v, want query recovery", result, err) + t.Fatalf("Execute() result=%#v error=%v, want same-invocation query recovery", result, err) + } + if !strings.Contains(result.Warning, "ambiguous") || !strings.Contains(result.Warning, "not replayed") { + t.Fatalf("Execute() warning = %q, want explicit safe-recovery message", result.Warning) } if api.applyCalls != 1 { t.Fatalf("committed apply was replayed: calls=%d", api.applyCalls) @@ -376,6 +394,9 @@ func TestExecutorResumesAcceptedCreateWithoutCreatingAgain(t *testing.T) { if err != nil || result.State != StateCreatePending || result.ProjectID == "" { t.Fatalf("first Execute() result=%#v error=%v", result, err) } + if result.DegradationCount != len(plan.Degradations) { + t.Fatalf("degradation count = %d, want %d", result.DegradationCount, len(plan.Degradations)) + } if api.createCalls != 1 || api.resumeCreateCalls != 0 { t.Fatalf("first call create=%d resume=%d", api.createCalls, api.resumeCreateCalls) } @@ -470,6 +491,7 @@ type fakeCanvasAPI struct { createPending bool createErr error getErr error + getNil bool applyErr error commitBeforeApplyError bool corruptGet bool @@ -524,6 +546,9 @@ func (api *fakeCanvasAPI) Get(_ context.Context, assetIDs []string) (*canvas.Get if api.getErr != nil { return nil, api.getErr } + if api.getNil { + return nil, nil + } assets := make([]json.RawMessage, 0, len(assetIDs)) for _, assetID := range assetIDs { content, ok := api.stored[assetID] diff --git a/internal/canvasplan/reconcile_test.go b/internal/canvasplan/reconcile_test.go new file mode 100644 index 0000000..da6dbdc --- /dev/null +++ b/internal/canvasplan/reconcile_test.go @@ -0,0 +1,425 @@ +package canvasplan + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/Pippit-dev/pippit-cli/internal/canvas" +) + +type reconcileTestAPI struct { + *fakeCanvasAPI + assets []json.RawMessage + requested []string +} + +func (api *reconcileTestAPI) Get(_ context.Context, assetIDs []string) (*canvas.GetResult, error) { + api.getCalls++ + api.requested = append([]string(nil), assetIDs...) + if api.getErr != nil { + return nil, api.getErr + } + return &canvas.GetResult{RequestedAssetIDs: assetIDs, Assets: api.assets, LogID: "reconcile-log"}, nil +} + +func TestReconcileVerifiesJournalWithoutApplying(t *testing.T) { + contents := map[string]json.RawMessage{ + "asset-b": json.RawMessage(`{"value":2}`), + "asset-a": json.RawMessage(`{"value":1}`), + } + journalPath := writeReconcileTestJournal(t, StateApplyAmbiguous, "ambiguous", contents) + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + api.assets = []json.RawMessage{ + queriedAsset(testedAsset{ID: "asset-b", Version: 2, Content: contents["asset-b"]}), + queriedAsset(testedAsset{ID: "asset-a", Version: 2, Content: contents["asset-a"]}), + } + + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if err != nil { + t.Fatalf("Reconcile() error = %v", err) + } + if result.State != StateVerified || result.Verification == nil || !result.Verification.Verified || !result.Verification.RecoveredFromQuery { + t.Fatalf("Reconcile() result = %#v, want verified query recovery", result) + } + if !reflect.DeepEqual(api.requested, []string{"asset-a", "asset-b"}) { + t.Fatalf("Get() asset IDs = %#v, want sorted journal keys", api.requested) + } + if api.applyCalls != 0 { + t.Fatalf("Reconcile() replayed Apply %d times", api.applyCalls) + } + if journal := readJournal(t, journalPath); journal.State != StateVerified || journal.Apply.Status != "verified" { + t.Fatalf("saved journal = %#v, want verified", journal) + } +} + +func TestReconcilePartialOrMismatchFailsClosed(t *testing.T) { + contents := map[string]json.RawMessage{ + "asset-a": json.RawMessage(`{"value":1}`), + "asset-b": json.RawMessage(`{"value":2}`), + } + journalPath := writeReconcileTestJournal(t, StateApplyAmbiguous, "ambiguous", contents) + beforeBytes, err := os.ReadFile(journalPath) + if err != nil { + t.Fatal(err) + } + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + api.assets = []json.RawMessage{ + queriedAsset(testedAsset{ID: "asset-a", Version: 2, Content: json.RawMessage(`{"value":99}`)}), + } + + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if err == nil || result == nil || result.State == StateVerified { + t.Fatalf("Reconcile() result=%#v error=%v, want nonverified failure", result, err) + } + if result.Verification == nil || len(result.Verification.MissingAssetIDs) != 1 || len(result.Verification.MismatchedAssetIDs) != 1 { + t.Fatalf("verification = %#v, want one missing and one mismatch", result.Verification) + } + if api.applyCalls != 0 { + t.Fatalf("Reconcile() replayed Apply %d times", api.applyCalls) + } + afterBytes, readErr := os.ReadFile(journalPath) + if readErr != nil { + t.Fatal(readErr) + } + if !reflect.DeepEqual(afterBytes, beforeBytes) { + t.Fatal("ambiguous journal bytes changed after non-exact query result") + } +} + +func TestReconcileRejectsInvalidStateWithoutQuery(t *testing.T) { + journalPath := writeReconcileTestJournal( + t, + StateApplyAcknowledged, + "acknowledged", + map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)}, + ) + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if err == nil || result == nil || !strings.Contains(err.Error(), "not eligible") { + t.Fatalf("Reconcile() result=%#v error=%v, want state rejection", result, err) + } + if api.getCalls != 0 || api.applyCalls != 0 { + t.Fatalf("invalid journal made remote calls: get=%d apply=%d", api.getCalls, api.applyCalls) + } +} + +func TestReconcilePreviouslyVerifiedCanvasAllowsRemoteEditsWithoutOverwritingHistory(t *testing.T) { + contents := map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)} + journalPath := writeReconcileTestJournal(t, StateVerified, "verified", contents) + before := readJournal(t, journalPath) + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + api.assets = []json.RawMessage{ + queriedAsset(testedAsset{ID: "asset-a", Version: 3, Content: json.RawMessage(`{"value":2}`)}), + } + + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if err != nil || result == nil || result.State != StateVerified || !strings.Contains(result.Warning, "changed after") { + t.Fatalf("Reconcile() result=%#v error=%v, want verified history with edit warning", result, err) + } + after := readJournal(t, journalPath) + if !reflect.DeepEqual(after.Verification, before.Verification) || after.State != StateVerified { + t.Fatalf("saved journal changed historical verification: before=%#v after=%#v", before, after) + } + if api.applyCalls != 0 { + t.Fatalf("Reconcile() replayed Apply %d times", api.applyCalls) + } +} + +func TestReconcilePreviouslyVerifiedCanvasAccessFailureDoesNotOverwriteHistory(t *testing.T) { + contents := map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)} + journalPath := writeReconcileTestJournal(t, StateVerified, "verified", contents) + before := readJournal(t, journalPath) + beforeBytes, err := os.ReadFile(journalPath) + if err != nil { + t.Fatal(err) + } + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + api.getErr = fmt.Errorf("not visible to current account") + + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if err == nil || result == nil || result.State != StateVerified || !strings.Contains(result.Warning, "current authentication") { + t.Fatalf("Reconcile() result=%#v error=%v, want access failure with verified history", result, err) + } + after := readJournal(t, journalPath) + if !reflect.DeepEqual(after.Verification, before.Verification) || after.State != StateVerified { + t.Fatalf("saved journal changed after access failure: before=%#v after=%#v", before, after) + } + afterBytes, readErr := os.ReadFile(journalPath) + if readErr != nil { + t.Fatal(readErr) + } + if !reflect.DeepEqual(afterBytes, beforeBytes) { + t.Fatal("journal bytes changed after current-auth query failure") + } +} + +func TestReconcileRejectsVerifiedJournalWithoutHistoricalVerification(t *testing.T) { + journalPath := writeReconcileTestJournal( + t, + StateVerified, + "verified", + map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)}, + ) + journal := readJournal(t, journalPath) + journal.Verification = &Verification{ExpectedAssetCount: 1, ReturnedAssetCount: 1} + if err := saveJournal(journalPath, journal); err != nil { + t.Fatal(err) + } + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if err == nil || result == nil || !strings.Contains(err.Error(), "historical verification") { + t.Fatalf("Reconcile() result=%#v error=%v, want corrupt verified journal rejection", result, err) + } + if api.getCalls != 0 || api.applyCalls != 0 { + t.Fatalf("corrupt verified journal made remote calls: get=%d apply=%d", api.getCalls, api.applyCalls) + } +} + +func TestReconcileRejectsInvalidInputHashesBeforeQuery(t *testing.T) { + for _, field := range []string{"plan", "resolved"} { + t.Run(field, func(t *testing.T) { + journalPath := writeReconcileTestJournal( + t, + StateVerified, + "verified", + map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)}, + ) + journal := readJournal(t, journalPath) + if field == "plan" { + journal.PlanSHA256 = "bad" + } else { + journal.ResolvedMediaSHA256 = "bad" + } + if err := saveJournal(journalPath, journal); err != nil { + t.Fatal(err) + } + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if err == nil || result == nil || !strings.Contains(err.Error(), "invalid plan or resolved-media") { + t.Fatalf("Reconcile() result=%#v error=%v, want invalid hash rejection", result, err) + } + if api.getCalls != 0 { + t.Fatalf("invalid hash made %d query call(s)", api.getCalls) + } + }) + } +} + +func TestReconcileRecoversLegacyDemotionAndPreservesStrictHistoryWhenPresent(t *testing.T) { + contents := map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)} + for _, historicalVerification := range []bool{false, true} { + t.Run(fmt.Sprintf("historical-verification-%t", historicalVerification), func(t *testing.T) { + journalPath := writeReconcileTestJournal(t, StateVerificationFailed, "verified", contents) + journal := readJournal(t, journalPath) + journal.LastError = "a later current-account query failed" + journal.Verification = &Verification{ + ExpectedAssetCount: 1, + ReturnedAssetCount: 1, + MismatchedAssetIDs: []string{"asset-a"}, + Verified: historicalVerification, + RecoveredFromQuery: true, + LogID: "old-query-log", + } + if historicalVerification { + journal.Verification.MismatchedAssetIDs = nil + } + if err := saveJournal(journalPath, journal); err != nil { + t.Fatal(err) + } + before := readJournal(t, journalPath) + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + api.assets = []json.RawMessage{ + queriedAsset(testedAsset{ID: "asset-a", Version: 3, Content: json.RawMessage(`{"value":2}`)}), + } + + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if err != nil || result == nil || result.State != StateVerified || + result.Verification == nil || !result.Verification.Verified || + !strings.Contains(result.Warning, "changed after") { + t.Fatalf("Reconcile() result=%#v error=%v, want accessible legacy recovery", result, err) + } + after := readJournal(t, journalPath) + if after.State != StateVerified || after.LastError != "" || !after.Verification.Verified { + t.Fatalf("saved journal = %#v, want repaired verified state", after) + } + if historicalVerification && !reflect.DeepEqual(after.Verification, before.Verification) { + t.Fatalf("historical strict verification was overwritten: before=%#v after=%#v", before.Verification, after.Verification) + } + if !historicalVerification && (!after.Verification.RecoveredFromQuery || after.Verification.LogID != "reconcile-log" || len(after.Verification.MismatchedAssetIDs) != 0) { + t.Fatalf("rebuilt completion marker = %#v, want auditable legacy recovery", after.Verification) + } + if api.applyCalls != 0 { + t.Fatalf("legacy recovery replayed Apply %d times", api.applyCalls) + } + }) + } +} + +func TestReconcileLegacyDemotionAccessFailureDoesNotRepairOrPersist(t *testing.T) { + journalPath := writeReconcileTestJournal( + t, + StateVerificationFailed, + "verified", + map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)}, + ) + journal := readJournal(t, journalPath) + journal.Verification = &Verification{ + ExpectedAssetCount: 1, + MissingAssetIDs: []string{"asset-a"}, + Verified: false, + RecoveredFromQuery: true, + } + if err := saveJournal(journalPath, journal); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(journalPath) + if err != nil { + t.Fatal(err) + } + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + api.getErr = errors.New("access key cannot see the imported assets") + + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if err == nil || result == nil || result.State != StateVerificationFailed { + t.Fatalf("Reconcile() result=%#v error=%v, want legacy access failure", result, err) + } + after, readErr := os.ReadFile(journalPath) + if readErr != nil { + t.Fatal(readErr) + } + if !reflect.DeepEqual(after, before) { + t.Fatal("legacy journal changed despite current-account access failure") + } +} + +func TestReconcileNormalVerificationFailureFallsBackWithoutQuery(t *testing.T) { + journalPath := writeReconcileTestJournal( + t, + StateVerificationFailed, + "verification-failed", + map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)}, + ) + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + result, err := (&Executor{api: api}).Reconcile(context.Background(), journalPath) + if !errors.Is(err, ErrReconcileNotEligible) || result == nil { + t.Fatalf("Reconcile() result=%#v error=%v, want normal Execute fallback signal", result, err) + } + if api.getCalls != 0 || api.applyCalls != 0 { + t.Fatalf("noneligible journal made remote calls: get=%d apply=%d", api.getCalls, api.applyCalls) + } +} + +func TestReconcileWithInputsRejectsDifferentPlanOrResolvedMediaBeforeQuery(t *testing.T) { + for _, change := range []string{"source", "resolved"} { + t.Run(change, func(t *testing.T) { + plan, resolved := testPlanAndResolved() + journalPath := writeReconcileTestJournal( + t, + StateVerified, + "verified", + map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)}, + ) + bindReconcileTestJournal(t, journalPath, plan, resolved) + if change == "source" { + plan.Source.ProjectID = "different-source-project" + } else { + resolved.Media[0].AssetID = "different-upload" + } + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + result, err := (&Executor{api: api}).ReconcileWithInputs(context.Background(), journalPath, plan, resolved) + if err == nil || result == nil || !strings.Contains(err.Error(), "changed after journal creation") { + t.Fatalf("ReconcileWithInputs() result=%#v error=%v, want input binding rejection", result, err) + } + if api.getCalls != 0 || api.applyCalls != 0 { + t.Fatalf("input mismatch made remote calls: get=%d apply=%d", api.getCalls, api.applyCalls) + } + }) + } +} + +func TestReconcileWithInputsReturnsPlanCounts(t *testing.T) { + plan, resolved := testPlanAndResolved() + contents := map[string]json.RawMessage{"asset-a": json.RawMessage(`{"value":1}`)} + journalPath := writeReconcileTestJournal(t, StateApplyAmbiguous, "ambiguous", contents) + bindReconcileTestJournal(t, journalPath, plan, resolved) + api := &reconcileTestAPI{fakeCanvasAPI: newFakeCanvasAPI(0)} + api.assets = []json.RawMessage{ + queriedAsset(testedAsset{ID: "asset-a", Version: 2, Content: contents["asset-a"]}), + } + + result, err := (&Executor{api: api}).ReconcileWithInputs(context.Background(), journalPath, plan, resolved) + if err != nil { + t.Fatalf("ReconcileWithInputs() error = %v", err) + } + if result.State != StateVerified || result.NodeCount != len(plan.Nodes)+len(plan.Groups) || + result.EdgeCount != len(plan.Edges) || result.DegradationCount != len(plan.Degradations) { + t.Fatalf("ReconcileWithInputs() result = %#v, want plan-derived counts", result) + } +} + +func bindReconcileTestJournal(t *testing.T, journalPath string, plan Plan, resolved ResolvedMediaSet) { + t.Helper() + normalizedPlan, err := NormalizePlan(plan) + if err != nil { + t.Fatal(err) + } + normalizedResolved, err := NormalizeResolvedMedia(resolved) + if err != nil { + t.Fatal(err) + } + journal := readJournal(t, journalPath) + journal.PlanSHA256, err = hashJSON(normalizedPlan) + if err != nil { + t.Fatal(err) + } + journal.ResolvedMediaSHA256, err = hashJSON(normalizedResolved) + if err != nil { + t.Fatal(err) + } + if err := saveJournal(journalPath, journal); err != nil { + t.Fatal(err) + } +} + +func writeReconcileTestJournal(t *testing.T, state, applyStatus string, contents map[string]json.RawMessage) string { + t.Helper() + hashes := make(map[string]string, len(contents)) + for assetID, content := range contents { + digest, err := hashRawJSON(content) + if err != nil { + t.Fatal(err) + } + hashes[assetID] = digest + } + path := filepath.Join(t.TempDir(), "reconcile.json") + journal := &Journal{ + Schema: JournalSchema, + OperationID: "operation-reconcile", + RequestID: "request-reconcile", + PlanSHA256: strings.Repeat("a", 64), + ResolvedMediaSHA256: strings.Repeat("b", 64), + State: state, + Create: &canvas.CreateResult{ProjectID: "123", CanvasAssetID: "asset-a", OverviewPippitAssetID: "overview", WebURL: "/novel/detail/canvas?projectId=123"}, + DocumentSHA256: fmt.Sprintf("%064x", 1), + AssetSHA256: hashes, + Apply: &ApplyJournal{TransactionID: "transaction-1", Status: applyStatus}, + } + if state == StateVerified { + journal.Verification = &Verification{ + ExpectedAssetCount: len(contents), + ReturnedAssetCount: len(contents), + Verified: true, + } + } + if err := saveJournal(path, journal); err != nil { + t.Fatal(err) + } + return path +} From ee755ab51de27093aadb158bceda69d9a2f05174 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 19:09:43 +0800 Subject: [PATCH 26/48] fix(canvas): resume imports without replaying writes Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import.go | 107 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 14 deletions(-) diff --git a/cmd/canvas/import.go b/cmd/canvas/import.go index 6bf64f4..352f937 100644 --- a/cmd/canvas/import.go +++ b/cmd/canvas/import.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "net/url" @@ -13,6 +14,7 @@ import ( "regexp" "runtime" "strings" + "time" "github.com/Pippit-dev/pippit-cli/internal/canvasplan" "github.com/Pippit-dev/pippit-cli/internal/common" @@ -24,9 +26,9 @@ type importOptions struct { SourceURL string Open bool AcceptDegradations bool + AcceptDegradationsExplicit bool JournalPath string OpenExplicit bool - AcceptDegradationsExplicit bool JournalExplicit bool } @@ -41,6 +43,7 @@ type importExporter interface { type importExecutor interface { Execute(context.Context, canvasplan.Plan, canvasplan.ResolvedMediaSet, canvasplan.ExecuteOptions) (*canvasplan.ExecutionResult, error) + Reconcile(context.Context, string, canvasplan.Plan, canvasplan.ResolvedMediaSet) (*canvasplan.ExecutionResult, error) } type importDependencies struct { @@ -53,6 +56,8 @@ type importDependencies struct { target func() string authScope func() string isInteractive func(io.Reader) bool + mediaPoll time.Duration + mediaTimeout time.Duration } type runnerImportExecutor struct { @@ -68,6 +73,15 @@ func (executor runnerImportExecutor) Execute( return executor.executor.Execute(ctx, plan, resolved, opts) } +func (executor runnerImportExecutor) Reconcile( + ctx context.Context, + journalPath string, + plan canvasplan.Plan, + resolved canvasplan.ResolvedMediaSet, +) (*canvasplan.ExecutionResult, error) { + return executor.executor.ReconcileWithInputs(ctx, journalPath, plan, resolved) +} + func newImportDependencies(runner *common.Runner) importDependencies { return importDependencies{ exporter: nodeLibTVExporter{}, @@ -79,6 +93,8 @@ func newImportDependencies(runner *common.Runner) importDependencies { target: func() string { return canvasImportTarget(runner) }, authScope: func() string { return canvasImportAuthScope(runner) }, isInteractive: importInputIsInteractive, + mediaPoll: defaultImportMediaPollInterval, + mediaTimeout: defaultImportMediaWaitTimeout, } } @@ -96,8 +112,8 @@ func newImportCommand( Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { opts.OpenExplicit = cmd.Flags().Changed("open") - opts.AcceptDegradationsExplicit = cmd.Flags().Changed("accept-degradations") opts.JournalExplicit = cmd.Flags().Changed("journal") + opts.AcceptDegradationsExplicit = cmd.Flags().Changed("accept-degradations") prepared, prompts, err := prepareCanvasImportOptions( cmd.InOrStdin(), opts, dependencies.isInteractive, stderr, ) @@ -174,20 +190,18 @@ func runCanvasImport( _ = removeOwnedBundle(outputDir, bundleRoot) return nil, err } - if len(plan.Degradations) > 0 && !opts.AcceptDegradations { - if prompts == nil || opts.AcceptDegradationsExplicit { - return nil, fmt.Errorf( - "LibTV export reports %d explicit degradation(s); inspect %s (plan: %s), then rerun with --accept-degradations", - len(plan.Degradations), outputDir, exported.PlanPath, + if len(plan.Degradations) > 0 { + if opts.AcceptDegradations { + // The caller explicitly accepted the adapter's auditable warnings. + } else if prompts != nil && !opts.AcceptDegradationsExplicit { + fmt.Fprintf( + stderr, + "Warning: LibTV export contains %d known nonfatal degradation(s), such as empty-media placeholders or semantic downgrades; continuing the interactive import. The final JSON records degradation_count.\n", + len(plan.Degradations), ) - } - accepted, promptErr := prompts.confirmDegradations(len(plan.Degradations)) - if promptErr != nil { - return nil, promptErr - } - if !accepted { + } else { return nil, fmt.Errorf( - "LibTV import was cancelled because the export contains %d degradation(s); inspect %s (plan: %s)", + "LibTV export reports %d explicit degradation(s); inspect %s (plan: %s), then rerun with --accept-degradations", len(plan.Degradations), outputDir, exported.PlanPath, ) } @@ -228,10 +242,23 @@ func runCanvasImport( BundleRoot: bundleRoot, CanvasJournalPath: journalPath, CheckpointPath: checkpointPath, + PollInterval: dependencies.mediaPoll, + WaitTimeout: dependencies.mediaTimeout, }, dependencies.media, stderr) if err != nil { return nil, err } + plan, err = canonicalizeImportPlanMedia(plan, target, journalPath, checkpointPath) + if err != nil { + return nil, fmt.Errorf("canonicalize CanvasPlan media identities: %w", err) + } + result, handled, reconcileErr := reconcileExistingCanvasImport( + ctx, journalPath, plan, resolved, opts, dependencies, stderr, + ) + if handled { + _ = removeOwnedBundle(outputDir, bundleRoot) + return result, reconcileErr + } fmt.Fprintln(stderr, "Phase canvas: create/resume, materialize, apply, then verify remote Canvas assets.") result, executeErr := dependencies.executor.Execute(ctx, plan, resolved, canvasplan.ExecuteOptions{ JournalPath: journalPath, @@ -239,6 +266,58 @@ func runCanvasImport( if executeErr != nil { return result, fmt.Errorf("execute CanvasPlan: %w", executeErr) } + return finishVerifiedCanvasImport(ctx, result, opts, dependencies, stderr) +} + +func reconcileExistingCanvasImport( + ctx context.Context, + journalPath string, + plan canvasplan.Plan, + resolved canvasplan.ResolvedMediaSet, + opts importOptions, + dependencies importDependencies, + stderr io.Writer, +) (*canvasplan.ExecutionResult, bool, error) { + info, err := os.Lstat(journalPath) + if os.IsNotExist(err) { + return nil, false, nil + } + if err != nil { + return nil, true, fmt.Errorf("inspect Canvas import journal for reconciliation: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return nil, true, fmt.Errorf("Canvas import journal must be a regular non-symbolic file") + } + result, reconcileErr := dependencies.executor.Reconcile(ctx, journalPath, plan, resolved) + if errors.Is(reconcileErr, canvasplan.ErrReconcileNotEligible) { + return nil, false, nil + } + if result != nil && !journalOnlyReconcileState(result.State) { + return nil, false, nil + } + if reconcileErr != nil { + return result, true, fmt.Errorf("reconcile Canvas import journal without replaying apply: %w", reconcileErr) + } + result, finishErr := finishVerifiedCanvasImport(ctx, result, opts, dependencies, stderr) + return result, true, finishErr +} + +func journalOnlyReconcileState(state string) bool { + switch state { + case canvasplan.StateApplyAmbiguous, canvasplan.StateVerified, canvasplan.StateVerificationFailed: + return true + default: + return false + } +} + +func finishVerifiedCanvasImport( + ctx context.Context, + result *canvasplan.ExecutionResult, + opts importOptions, + dependencies importDependencies, + stderr io.Writer, +) (*canvasplan.ExecutionResult, error) { if !verifiedExecution(result) { return result, fmt.Errorf("CanvasPlan execution completed without query-back verification") } From 5a453724bdd4e44f40314ede1403b2a00987ca50 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 19:10:02 +0800 Subject: [PATCH 27/48] feat(canvas): guide interactive imports Co-authored-by: Codex <codex@openai.com> --- README.md | 2 +- cmd/canvas/import_prompt.go | 110 ++++++++++++++++++-------- cmd/canvas/import_terminal_darwin.go | 17 ++++ cmd/canvas/import_terminal_linux.go | 17 ++++ cmd/canvas/import_terminal_windows.go | 17 ++++ go.mod | 2 +- 6 files changed, 131 insertions(+), 34 deletions(-) create mode 100644 cmd/canvas/import_terminal_darwin.go create mode 100644 cmd/canvas/import_terminal_linux.go create mode 100644 cmd/canvas/import_terminal_windows.go diff --git a/README.md b/README.md index 4398979..344b898 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ LibTV 迁移只是上述通用画布能力的 CLI 编排层，服务端不识别 pippit-tool-cli --ppe-env ppe_cli_canvas_ak canvas import ``` -CLI 会逐步询问来源、LibTV 链接、journal 位置、降级接受与是否打开结果。journal 直接回车即使用权限受控的自动路径，不需要设置环境变量。源端节点处理、素材下载与 Pippit 素材上传会在 stderr 显示已处理/总数/剩余数，画布创建、写入和回读校验会显示当前阶段；最终 stdout 仍只输出一行 JSON。 +CLI 会用编号选项逐步询问来源、journal 策略与是否打开结果，只有每个项目唯一的 LibTV 链接需要粘贴。journal 选择 Automatic 即使用权限受控的自动路径，不需要设置环境变量。交互式导入遇到已知的非致命降级时会输出 warning 后自动继续，并在最终 JSON 中保留 `degradation_count`。源端节点处理、素材下载与 Pippit 素材上传会在 stderr 显示已处理/总数/剩余数，画布创建、写入和回读校验会显示当前阶段；最终 stdout 仍只输出一行 JSON。 供 Agent、CI 或其它非交互场景使用时，仍可显式传入 `--from`、`--url`、`--accept-degradations` 和 `--open`；`--journal` 始终可选，省略时使用自动路径。 diff --git a/cmd/canvas/import_prompt.go b/cmd/canvas/import_prompt.go index f4e6610..880480a 100644 --- a/cmd/canvas/import_prompt.go +++ b/cmd/canvas/import_prompt.go @@ -16,13 +16,17 @@ type importPromptSession struct { eof bool } +type importPromptChoice struct { + label string + aliases []string +} + func importInputIsInteractive(input io.Reader) bool { file, ok := input.(*os.File) if !ok { return false } - info, err := file.Stat() - return err == nil && info.Mode()&os.ModeCharDevice != 0 + return importFileIsTerminal(file) } func prepareCanvasImportOptions( @@ -43,14 +47,15 @@ func prepareCanvasImportOptions( } prompts := &importPromptSession{reader: bufio.NewReader(input), stderr: stderr} if strings.TrimSpace(opts.Provider) == "" { - value, _, err := prompts.readLine("Source provider [libtv]: ") + _, err := prompts.askChoice( + "Source provider:", + []importPromptChoice{{label: "LibTV (default)", aliases: []string{"libtv"}}}, + 1, + ) if err != nil { return opts, nil, err } - if value == "" { - value = "libtv" - } - opts.Provider = value + opts.Provider = "libtv" } if strings.TrimSpace(opts.SourceURL) == "" { for { @@ -72,50 +77,91 @@ func prepareCanvasImportOptions( } } if !opts.JournalExplicit { - value, _, err := prompts.readLine("Resume journal path [automatic]: ") + choice, err := prompts.askChoice( + "Resume journal:", + []importPromptChoice{ + {label: "Automatic (recommended, default)"}, + {label: "Custom path"}, + }, + 1, + ) if err != nil { return opts, nil, err } - if value != "" { - opts.JournalPath = value - opts.JournalExplicit = true + if choice == 2 { + for { + value, eof, readErr := prompts.readLine("Custom journal path: ") + if readErr != nil { + return opts, nil, readErr + } + if value != "" { + opts.JournalPath = value + opts.JournalExplicit = true + break + } + if eof { + return opts, nil, fmt.Errorf( + "interactive input ended before a custom journal path was provided; choose 1 for Automatic or pass --journal <path>", + ) + } + fmt.Fprintln(stderr, "A custom journal path is required after selecting option 2.") + } } } if !opts.OpenExplicit { - open, err := prompts.askYesNo("Open the imported Canvas when finished? [Y/n]: ", true) + choice, err := prompts.askChoice( + "After import:", + []importPromptChoice{ + {label: "Open Canvas (default)", aliases: []string{"y", "yes"}}, + {label: "Do not open", aliases: []string{"n", "no"}}, + }, + 1, + ) if err != nil { return opts, nil, err } - opts.Open = open + opts.Open = choice == 1 } return opts, prompts, nil } -func (prompts *importPromptSession) confirmDegradations(count int) (bool, error) { - fmt.Fprintf(prompts.stderr, "LibTV export reports %d explicit degradation(s).\n", count) - return prompts.askYesNo("Continue importing with these degradations? [y/N]: ", false) -} - -func (prompts *importPromptSession) askYesNo(label string, defaultValue bool) (bool, error) { +func (prompts *importPromptSession) askChoice( + title string, + choices []importPromptChoice, + defaultChoice int, +) (int, error) { for { - value, eof, err := prompts.readLine(label) + fmt.Fprintln(prompts.stderr, title) + for index, choice := range choices { + fmt.Fprintf(prompts.stderr, " %d) %s\n", index+1, choice.label) + } + value, eof, err := prompts.readLine(fmt.Sprintf("Select [%d]: ", defaultChoice)) if err != nil { - return false, err + return 0, err } - switch strings.ToLower(value) { - case "": - return defaultValue, nil - case "y", "yes": - return true, nil - case "n", "no": - return false, nil - default: - if eof { - return defaultValue, nil + if value == "" { + return defaultChoice, nil + } + normalized := strings.ToLower(value) + for index, choice := range choices { + if normalized == fmt.Sprint(index+1) || containsImportPromptAlias(choice.aliases, normalized) { + return index + 1, nil } - fmt.Fprintln(prompts.stderr, "Please answer y or n.") + } + if eof { + return defaultChoice, nil + } + fmt.Fprintf(prompts.stderr, "Please select a number from 1 to %d.\n", len(choices)) + } +} + +func containsImportPromptAlias(values []string, expected string) bool { + for _, value := range values { + if value == expected { + return true } } + return false } func (prompts *importPromptSession) readLine(label string) (string, bool, error) { diff --git a/cmd/canvas/import_terminal_darwin.go b/cmd/canvas/import_terminal_darwin.go new file mode 100644 index 0000000..0c2fa50 --- /dev/null +++ b/cmd/canvas/import_terminal_darwin.go @@ -0,0 +1,17 @@ +//go:build darwin + +package canvas + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func importFileIsTerminal(file *os.File) bool { + if file == nil { + return false + } + _, err := unix.IoctlGetTermios(int(file.Fd()), uint(unix.TIOCGETA)) + return err == nil +} diff --git a/cmd/canvas/import_terminal_linux.go b/cmd/canvas/import_terminal_linux.go new file mode 100644 index 0000000..a32429e --- /dev/null +++ b/cmd/canvas/import_terminal_linux.go @@ -0,0 +1,17 @@ +//go:build linux + +package canvas + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func importFileIsTerminal(file *os.File) bool { + if file == nil { + return false + } + _, err := unix.IoctlGetTermios(int(file.Fd()), uint(unix.TCGETS)) + return err == nil +} diff --git a/cmd/canvas/import_terminal_windows.go b/cmd/canvas/import_terminal_windows.go new file mode 100644 index 0000000..c6a73d5 --- /dev/null +++ b/cmd/canvas/import_terminal_windows.go @@ -0,0 +1,17 @@ +//go:build windows + +package canvas + +import ( + "os" + + "golang.org/x/sys/windows" +) + +func importFileIsTerminal(file *os.File) bool { + if file == nil { + return false + } + var mode uint32 + return windows.GetConsoleMode(windows.Handle(file.Fd()), &mode) == nil +} diff --git a/go.mod b/go.mod index a9f5bb5..fe5d4d5 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23 require ( github.com/bytedance/sonic v1.15.1 github.com/spf13/cobra v1.8.1 + golang.org/x/sys v0.27.0 ) require ( @@ -17,5 +18,4 @@ require ( github.com/stretchr/testify v1.11.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect - golang.org/x/sys v0.27.0 // indirect ) From 873824d76b8f3c76f9f191a00537f02338a40b82 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 19:10:15 +0800 Subject: [PATCH 28/48] test(canvas): cover resilient one-shot imports Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_media_fingerprint_test.go | 254 ++++++++ cmd/canvas/import_test.go | 618 ++++++++++++++++++-- 2 files changed, 830 insertions(+), 42 deletions(-) create mode 100644 cmd/canvas/import_media_fingerprint_test.go diff --git a/cmd/canvas/import_media_fingerprint_test.go b/cmd/canvas/import_media_fingerprint_test.go new file mode 100644 index 0000000..132e33d --- /dev/null +++ b/cmd/canvas/import_media_fingerprint_test.go @@ -0,0 +1,254 @@ +package canvas + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "image/color" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +type generatedImportMediaReader struct { + remaining int64 + maxReadRequest int +} + +type verifiedReaderUploadClient struct { + path string + wantBytes []byte + uploadRead bool +} + +func (client *verifiedReaderUploadClient) SendRequest(_ context.Context, _ string, _ any, out any) error { + return json.Unmarshal([]byte(`{"ret":"0","data":{"Assets":[{"PippitAssetID":"pippit-1"}]}}`), out) +} + +func (client *verifiedReaderUploadClient) SendRequestWithHeaders( + ctx context.Context, + path string, + body any, + _ map[string]string, + out any, +) error { + return client.SendRequest(ctx, path, body, out) +} + +func (client *verifiedReaderUploadClient) SendMultipartRequest( + _ context.Context, + _ string, + _ map[string]string, + file common.MultipartFile, + out any, +) error { + if file.Reader == nil || file.Path != "" { + return fmt.Errorf("multipart must use the caller-verified reader without a fallback path") + } + if err := os.Rename(client.path, client.path+".replaced"); err != nil { + return err + } + if err := os.WriteFile(client.path, []byte("attacker replacement"), 0o600); err != nil { + return err + } + payload, err := io.ReadAll(file.Reader) + if err != nil { + return err + } + if !bytes.Equal(payload, client.wantBytes) { + return fmt.Errorf("uploaded payload %q does not match verified inode", payload) + } + client.uploadRead = true + return json.Unmarshal([]byte(`{"ret":"0","data":{"asset_id":"asset-1","pippit_asset_id":"pippit-1"}}`), out) +} + +func (reader *generatedImportMediaReader) Read(payload []byte) (int, error) { + if len(payload) > reader.maxReadRequest { + reader.maxReadRequest = len(payload) + } + if reader.remaining == 0 { + return 0, io.EOF + } + if int64(len(payload)) > reader.remaining { + payload = payload[:reader.remaining] + } + for index := range payload { + payload[index] = 0x5a + } + reader.remaining -= int64(len(payload)) + return len(payload), nil +} + +func TestInspectImportMediaContentStreamsNonPNG(t *testing.T) { + const byteSize = int64(8 << 20) + reader := &generatedImportMediaReader{remaining: byteSize} + identity, err := inspectImportMediaContent(reader) + if err != nil { + t.Fatalf("inspectImportMediaContent() error = %v", err) + } + if identity.ByteSize != byteSize || !strings.HasPrefix(identity.ContentFingerprint, rawMediaFingerprintPrefix) || + identity.ContentFingerprint != rawMediaFingerprintPrefix+identity.RawSHA256 { + t.Fatalf("identity = %#v, want streamed raw identity", identity) + } + if reader.maxReadRequest > 64<<10 { + t.Fatalf("largest Read buffer = %d, want bounded streaming instead of whole-file allocation", reader.maxReadRequest) + } +} + +func TestLibTVPNGAIGCFingerprintFallsBackForUnrecognizedSchema(t *testing.T) { + known := testLibTVAIGCMetadata("libtv" + strings.Repeat("a", 32)) + base := testPNGWithITXt(t, testFingerprintPixel, known) + secondAIGC := append([]byte("AIGC\x00\x00\x00\x00\x00"), []byte(known)...) + oversizedAIGC := append( + []byte("AIGC\x00\x00\x00\x00\x00"), + bytes.Repeat([]byte{'x'}, maxCanonicalITXtChunkBytes+1)..., + ) + cases := map[string][]byte{ + "missing ID": testPNGWithITXt(t, testFingerprintPixel, `{"ProduceID":"one"}`), + "non-string ID": testPNGWithITXt( + t, testFingerprintPixel, `{"ProduceID":1,"PropagateID":"two"}`, + ), + "unknown extra field": testPNGWithITXt( + t, testFingerprintPixel, strings.TrimSuffix(known, "}")+`,"Prompt":"future"}`, + ), + "future compressed schema": rewriteTestITXtHeader(t, base, func(header []byte) { + header[len("AIGC")+1] = 1 + }), + "future language schema": rewriteTestITXtHeader(t, base, func(header []byte) { + header[len("AIGC")+3] = 'e' + }), + "multiple AIGC chunks": testInsertPNGChunkBeforeIEND(t, base, "iTXt", secondAIGC), + "oversized second AIGC chunk": testInsertPNGChunkBeforeIEND( + t, base, "iTXt", oversizedAIGC, + ), + } + for name, payload := range cases { + t.Run(name, func(t *testing.T) { + fingerprint, err := canonicalLibTVPNGFingerprint(payload) + if err != nil { + t.Fatalf("canonicalLibTVPNGFingerprint() error = %v", err) + } + digest := sha256.Sum256(payload) + want := rawMediaFingerprintPrefix + hex.EncodeToString(digest[:]) + if fingerprint != want { + t.Fatalf("fingerprint = %q, want raw fallback %q", fingerprint, want) + } + }) + } +} + +func TestInspectImportMediaFileRejectsFinalSymlink(t *testing.T) { + directory := t.TempDir() + target := filepath.Join(directory, "target.bin") + link := filepath.Join(directory, "link.bin") + if err := os.WriteFile(target, []byte("media"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, link); err != nil { + t.Skipf("create symlink: %v", err) + } + if _, err := inspectImportMediaFile(link); err == nil || !strings.Contains(err.Error(), "non-symbolic") { + t.Fatalf("inspectImportMediaFile() error = %v, want no-follow rejection", err) + } +} + +func TestRunnerImportMediaUploadStreamsVerifiedInode(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "clip.mp4") + original := []byte("verified original media") + if err := os.WriteFile(path, original, 0o600); err != nil { + t.Fatal(err) + } + identity, err := inspectImportMediaFile(path) + if err != nil { + t.Fatal(err) + } + client := &verifiedReaderUploadClient{path: path, wantBytes: original} + api := runnerImportMediaAPI{runner: &common.Runner{Client: client}} + result, err := api.Upload(context.Background(), validatedImportMedia{ + LogicalID: "media:video-1", + MediaType: "video", + FileName: "clip.mp4", + LocalPath: path, + SHA256: identity.RawSHA256, + ContentFingerprint: identity.ContentFingerprint, + ByteSize: identity.ByteSize, + }) + if err != nil { + t.Fatalf("Upload() error = %v", err) + } + if !client.uploadRead || result.PippitAssetID != "pippit-1" { + t.Fatalf("uploadRead/result = %v/%#v, want verified descriptor upload", client.uploadRead, result) + } +} + +func TestLegacyPNGCheckpointBindsSourceNodeRequirement(t *testing.T) { + oldPNG := testPNGWithITXt(t, testFingerprintPixel, testLibTVAIGCMetadata("libtv"+strings.Repeat("a", 32))) + currentPNG := testPNGWithITXt(t, testFingerprintPixel, testLibTVAIGCMetadata("libtv"+strings.Repeat("b", 32))) + opts, _ := testPNGCheckpointMigrationOptions(t, oldPNG, currentPNG) + checkpoint := readTestMediaCheckpoint(t, opts.CheckpointPath) + planPath := filepath.Join(checkpoint.BundleDirs[0], "plan.json") + plan, err := readCanvasPlan(planPath) + if err != nil { + t.Fatal(err) + } + plan.RequiredMedia[0].SourceNodeID = "different-source-node" + if err := writeTestJSON(planPath, plan); err != nil { + t.Fatal(err) + } + api := &fakeImportMediaAPI{} + _, err = resolveImportMedia(context.Background(), opts, api, io.Discard) + if err == nil || !strings.Contains(err.Error(), "source node and media contract") { + t.Fatalf("resolveImportMedia() error = %v, want source requirement rejection", err) + } + if api.uploads != 0 || api.queries != 0 { + t.Fatalf("uploads/queries = %d/%d, want no remote calls", api.uploads, api.queries) + } +} + +func TestLegacyPNGCheckpointRejectsMultipleMatchingBundles(t *testing.T) { + oldPNG := testPNGWithITXt(t, testFingerprintPixel, testLibTVAIGCMetadata("libtv"+strings.Repeat("a", 32))) + currentPNG := testPNGWithITXt(t, testFingerprintPixel, testLibTVAIGCMetadata("libtv"+strings.Repeat("b", 32))) + opts, _ := testPNGCheckpointMigrationOptions(t, oldPNG, currentPNG) + checkpoint := readTestMediaCheckpoint(t, opts.CheckpointPath) + oldPlan, err := readCanvasPlan(filepath.Join(checkpoint.BundleDirs[0], "plan.json")) + if err != nil { + t.Fatal(err) + } + duplicateBundle := filepath.Join(opts.BundleRoot, "export-old-duplicate") + exporter := &fakeImportExporter{ + plan: oldPlan, + mediaBytes: map[string][]byte{ + oldPlan.RequiredMedia[0].LocalPath: oldPNG, + }, + } + if _, err := exporter.Export(context.Background(), testLibTVURL, duplicateBundle, io.Discard); err != nil { + t.Fatal(err) + } + checkpoint.BundleDirs = append(checkpoint.BundleDirs, duplicateBundle) + if err := saveMediaCheckpoint(opts.CheckpointPath, &checkpoint); err != nil { + t.Fatal(err) + } + api := &fakeImportMediaAPI{} + _, err = resolveImportMedia(context.Background(), opts, api, io.Discard) + if err == nil || !strings.Contains(err.Error(), "ambiguously match") { + t.Fatalf("resolveImportMedia() error = %v, want ambiguous legacy bundle rejection", err) + } + if api.uploads != 0 || api.queries != 0 { + t.Fatalf("uploads/queries = %d/%d, want no remote calls", api.uploads, api.queries) + } +} + +var testFingerprintPixel = color.NRGBA{R: 20, G: 40, B: 60, A: 255} + +func rewriteTestITXtHeader(t *testing.T, payload []byte, rewrite func([]byte)) []byte { + t.Helper() + return testRewriteFirstPNGChunk(t, payload, "iTXt", rewrite) +} diff --git a/cmd/canvas/import_test.go b/cmd/canvas/import_test.go index 9557ae8..3ab50fe 100644 --- a/cmd/canvas/import_test.go +++ b/cmd/canvas/import_test.go @@ -4,14 +4,21 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/binary" "encoding/hex" "encoding/json" "errors" + "fmt" + "hash/crc32" + "image" + "image/color" + "image/png" "io" "os" "path/filepath" "strings" "testing" + "time" canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" "github.com/Pippit-dev/pippit-cli/internal/canvasplan" @@ -88,9 +95,10 @@ type fakeImportMediaAPI struct { uploadState string uploadErr error queryErr error + queryReady []bool } -func (api *fakeImportMediaAPI) Upload(_ context.Context, _ string) (*canvascore.UploadResult, error) { +func (api *fakeImportMediaAPI) Upload(_ context.Context, _ validatedImportMedia) (*canvascore.UploadResult, error) { api.uploads++ if api.uploadErr != nil { return nil, api.uploadErr @@ -104,22 +112,30 @@ func (api *fakeImportMediaAPI) Upload(_ context.Context, _ string) (*canvascore. }, nil } -func (api *fakeImportMediaAPI) Query(context.Context, string) error { +func (api *fakeImportMediaAPI) Query(context.Context, string) (bool, error) { api.queries++ - return api.queryErr + if api.queryErr != nil { + return false, api.queryErr + } + if len(api.queryReady) == 0 { + return true, nil + } + ready := api.queryReady[0] + api.queryReady = api.queryReady[1:] + return ready, nil } type panickingImportMediaAPI struct { uploads int } -func (api *panickingImportMediaAPI) Upload(context.Context, string) (*canvascore.UploadResult, error) { +func (api *panickingImportMediaAPI) Upload(context.Context, validatedImportMedia) (*canvascore.UploadResult, error) { api.uploads++ panic("simulated process exit during upload") } -func (*panickingImportMediaAPI) Query(context.Context, string) error { - return nil +func (*panickingImportMediaAPI) Query(context.Context, string) (bool, error) { + return true, nil } type missingAKPreflightMediaAPI struct { @@ -130,13 +146,13 @@ func (*missingAKPreflightMediaAPI) PreflightUpload(context.Context) error { return errors.New("XYQ_ACCESS_KEY 缺失") } -func (api *missingAKPreflightMediaAPI) Upload(context.Context, string) (*canvascore.UploadResult, error) { +func (api *missingAKPreflightMediaAPI) Upload(context.Context, validatedImportMedia) (*canvascore.UploadResult, error) { api.uploads++ return nil, errors.New("must not be called") } -func (*missingAKPreflightMediaAPI) Query(context.Context, string) error { - return nil +func (*missingAKPreflightMediaAPI) Query(context.Context, string) (bool, error) { + return true, nil } type blockingImportMediaAPI struct { @@ -145,7 +161,7 @@ type blockingImportMediaAPI struct { uploads int } -func (api *blockingImportMediaAPI) Upload(context.Context, string) (*canvascore.UploadResult, error) { +func (api *blockingImportMediaAPI) Upload(context.Context, validatedImportMedia) (*canvascore.UploadResult, error) { api.uploads++ close(api.started) <-api.release @@ -154,15 +170,21 @@ func (api *blockingImportMediaAPI) Upload(context.Context, string) (*canvascore. }, nil } -func (*blockingImportMediaAPI) Query(context.Context, string) error { - return nil +func (*blockingImportMediaAPI) Query(context.Context, string) (bool, error) { + return true, nil } type fakeImportExecutor struct { - calls int - resolved canvasplan.ResolvedMediaSet - opts canvasplan.ExecuteOptions - result *canvasplan.ExecutionResult + calls int + reconcileCalls int + resolved canvasplan.ResolvedMediaSet + opts canvasplan.ExecuteOptions + result *canvasplan.ExecutionResult + reconcileJournal string + reconcilePlan canvasplan.Plan + reconcileResolved canvasplan.ResolvedMediaSet + reconcileResult *canvasplan.ExecutionResult + reconcileErr error } func (executor *fakeImportExecutor) Execute( @@ -177,6 +199,19 @@ func (executor *fakeImportExecutor) Execute( return executor.result, nil } +func (executor *fakeImportExecutor) Reconcile( + _ context.Context, + journalPath string, + plan canvasplan.Plan, + resolved canvasplan.ResolvedMediaSet, +) (*canvasplan.ExecutionResult, error) { + executor.reconcileCalls++ + executor.reconcileJournal = journalPath + executor.reconcilePlan = plan + executor.reconcileResolved = resolved + return executor.reconcileResult, executor.reconcileErr +} + func TestImportCommandExportsUploadsDeduplicatesVerifiesAndOpens(t *testing.T) { temp := t.TempDir() plan, mediaBytes := testImportPlan(t, false) @@ -221,6 +256,32 @@ func TestImportCommandExportsUploadsDeduplicatesVerifiesAndOpens(t *testing.T) { } } +func TestImportCommandFallsBackToExecuteForNormalVerificationFailure(t *testing.T) { + temp := t.TempDir() + _, exporter, journalPath := prepareSourceBoundResumeFixture(t, temp) + executor := &fakeImportExecutor{ + result: verifiedImportResult(), + reconcileResult: &canvasplan.ExecutionResult{State: canvasplan.StateVerificationFailed}, + reconcileErr: canvasplan.ErrReconcileNotEligible, + } + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + cmd := newImportCommand(io.Discard, io.Discard, deps) + cmd.SilenceUsage = true + cmd.SetArgs([]string{ + "--from", "libtv", "--url", testLibTVURL, "--journal", journalPath, + "--accept-degradations", + }) + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v, want normal executor fallback", err) + } + if executor.reconcileCalls != 1 || executor.calls != 1 { + t.Fatalf( + "input-reconcile/execute calls = %d/%d, want 1/1", + executor.reconcileCalls, executor.calls, + ) + } +} + func TestImportCommandInteractiveWizardUsesSafeDefaults(t *testing.T) { temp := t.TempDir() plan, mediaBytes := testImportPlan(t, false) @@ -246,8 +307,9 @@ func TestImportCommandInteractiveWizardUsesSafeDefaults(t *testing.T) { t.Fatalf("opened = %q, want wizard default Yes", opened) } for _, message := range []string{ - "Source provider [libtv]", "LibTV canvas URL", "Resume journal path [automatic]", - "Open the imported Canvas when finished? [Y/n]", + "Source provider:", "1) LibTV (default)", "LibTV canvas URL", "Resume journal:", + "1) Automatic (recommended, default)", "2) Custom path", "After import:", + "1) Open Canvas (default)", "2) Do not open", "Resume journal: " + executor.opts.JournalPath, `Media progress: processed=1/2 remaining=1 action=uploaded file="one.png"`, `Media progress: processed=2/2 remaining=0 action=reused file="two.png"`, @@ -264,7 +326,44 @@ func TestImportCommandInteractiveWizardUsesSafeDefaults(t *testing.T) { } } -func TestImportCommandInteractiveWizardCanAcceptDegradations(t *testing.T) { +func TestImportCommandInteractiveWizardRetriesAndUsesNumberedCustomChoices(t *testing.T) { + temp := t.TempDir() + journalDirectory := filepath.Join(temp, "custom-state") + if err := os.MkdirAll(journalDirectory, 0o700); err != nil { + t.Fatal(err) + } + journalPath := filepath.Join(journalDirectory, "import.journal.json") + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + executor := &fakeImportExecutor{result: verifiedImportResult()} + opened := false + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + deps.isInteractive = func(io.Reader) bool { return true } + deps.openURL = func(context.Context, string) error { opened = true; return nil } + var stdout, stderr bytes.Buffer + cmd := newImportCommand(&stdout, &stderr, deps) + cmd.SetIn(strings.NewReader(strings.Join([]string{ + "9", "1", testLibTVURL, "9", "2", journalPath, "maybe", "2", "", + }, "\n"))) + cmd.SilenceUsage = true + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v, stderr = %s", err, stderr.String()) + } + if executor.opts.JournalPath != journalPath { + t.Fatalf("journal = %q, want custom path %q", executor.opts.JournalPath, journalPath) + } + if opened { + t.Fatal("wizard option 2 unexpectedly opened the Canvas") + } + if got := strings.Count(stderr.String(), "Please select a number from 1 to"); got != 3 { + t.Fatalf("invalid choice messages = %d, want 3:\n%s", got, stderr.String()) + } + if !json.Valid(bytes.TrimSpace(stdout.Bytes())) { + t.Fatalf("stdout = %q, want final JSON", stdout.String()) + } +} + +func TestImportCommandInteractiveWizardWarnsAndContinuesDegradations(t *testing.T) { temp := t.TempDir() plan, mediaBytes := testImportPlan(t, true) exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} @@ -273,19 +372,64 @@ func TestImportCommandInteractiveWizardCanAcceptDegradations(t *testing.T) { deps.isInteractive = func(io.Reader) bool { return true } var stdout, stderr bytes.Buffer cmd := newImportCommand(&stdout, &stderr, deps) - cmd.SetIn(strings.NewReader("\n" + testLibTVURL + "\n\nn\ny\n")) + cmd.SetIn(strings.NewReader("\n" + testLibTVURL + "\n\nn\n")) cmd.SilenceUsage = true if err := cmd.Execute(); err != nil { t.Fatalf("Execute() error = %v, stderr = %s", err, stderr.String()) } - if !strings.Contains(stderr.String(), "Continue importing with these degradations? [y/N]") { - t.Fatalf("stderr = %q, want in-session degradation confirmation", stderr.String()) + if !strings.Contains(stderr.String(), "known nonfatal degradation(s)") || + !strings.Contains(stderr.String(), "empty-media placeholders or semantic downgrades") || + !strings.Contains(stderr.String(), "degradation_count") { + t.Fatalf("stderr = %q, want auditable automatic degradation warning", stderr.String()) } if executor.calls != 1 || !json.Valid(bytes.TrimSpace(stdout.Bytes())) { t.Fatalf("executor/stdout = %d/%q, want completed interactive import", executor.calls, stdout.String()) } } +func TestImportCommandInteractiveWizardHonorsExplicitDegradationRejection(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, true) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + executor := &fakeImportExecutor{result: verifiedImportResult()} + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + deps.isInteractive = func(io.Reader) bool { return true } + var stderr bytes.Buffer + cmd := newImportCommand(io.Discard, &stderr, deps) + cmd.SetIn(strings.NewReader("\n" + testLibTVURL + "\n\n\n")) + cmd.SetArgs([]string{"--accept-degradations=false"}) + cmd.SilenceUsage = true + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "--accept-degradations") { + t.Fatalf("Execute() error = %v, want explicit degradation rejection", err) + } + if executor.calls != 0 { + t.Fatalf("executor calls = %d, want no Canvas write after explicit rejection", executor.calls) + } + if strings.Contains(stderr.String(), "continuing the interactive import") { + t.Fatalf("stderr = %q, explicit false must not be ignored by the wizard", stderr.String()) + } +} + +func TestImportCommandInteractiveCustomJournalEOFIsActionable(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, &fakeImportExecutor{}) + deps.isInteractive = func(io.Reader) bool { return true } + cmd := newImportCommand(io.Discard, io.Discard, deps) + cmd.SetIn(strings.NewReader("\n" + testLibTVURL + "\n2\n")) + cmd.SilenceUsage = true + err := cmd.Execute() + if err == nil || !strings.Contains(err.Error(), "custom journal path") || + !strings.Contains(err.Error(), "choose 1 for Automatic") { + t.Fatalf("Execute() error = %v, want actionable custom path EOF", err) + } + if len(exporter.urls) != 0 { + t.Fatalf("exporter called before custom journal input completed: %#v", exporter.urls) + } +} + func TestImportCommandMissingFlagsFailsActionablyWithoutInteractiveInput(t *testing.T) { temp := t.TempDir() plan, mediaBytes := testImportPlan(t, false) @@ -304,6 +448,17 @@ func TestImportCommandMissingFlagsFailsActionablyWithoutInteractiveInput(t *test } } +func TestImportInputDoesNotTreatNullDeviceAsInteractive(t *testing.T) { + input, err := os.Open(os.DevNull) + if err != nil { + t.Fatal(err) + } + defer input.Close() + if importInputIsInteractive(input) { + t.Fatal("null device was incorrectly treated as an interactive terminal") + } +} + func TestImportCommandInteractiveEOFMissingURLFailsBeforeExport(t *testing.T) { temp := t.TempDir() plan, mediaBytes := testImportPlan(t, false) @@ -386,36 +541,248 @@ func TestImportCommandRequiresExplicitDegradationAcceptanceAndKeepsBundle(t *tes } } -func TestImportCommandCheckpointsProcessingUploadAndDoesNotUploadAgain(t *testing.T) { +func TestImportCommandWaitsForProcessingUploadAndContinuesSameInvocation(t *testing.T) { temp := t.TempDir() plan, mediaBytes := testSingleMediaPlan(t) exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} - media := &fakeImportMediaAPI{uploadState: canvascore.StateProcessing} + media := &fakeImportMediaAPI{ + uploadState: canvascore.StateProcessing, + queryReady: []bool{false, false, true}, + } executor := &fakeImportExecutor{result: verifiedImportResult()} deps := testImportDependencies(temp, exporter, media, executor) var stdout, stderr bytes.Buffer - first := newImportCommand(&stdout, &stderr, deps) - first.SilenceUsage = true - first.SetArgs([]string{"--from", "libtv", "--url", testLibTVURL}) - if err := first.Execute(); err == nil || !strings.Contains(err.Error(), "still processing") { - t.Fatalf("first Execute() error = %v, want processing checkpoint", err) + cmd := newImportCommand(&stdout, &stderr, deps) + cmd.SilenceUsage = true + cmd.SetArgs([]string{"--from", "libtv", "--url", testLibTVURL}) + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v, stderr = %s", err, stderr.String()) + } + if media.uploads != 1 || media.queries != 3 || executor.calls != 1 { + t.Fatalf("upload/query/execute = %d/%d/%d, want 1/3/1 in one invocation", media.uploads, media.queries, executor.calls) + } + for _, progress := range []string{ + `Media progress: processed=0/1 remaining=1 action=processing file="one.png"`, + `Media progress: processed=0/1 remaining=1 action=waiting file="one.png"`, + `Media progress: processed=1/1 remaining=0 action=uploaded file="one.png"`, + } { + if !strings.Contains(stderr.String(), progress) { + t.Fatalf("stderr = %q, want progress %q", stderr.String(), progress) + } + } + if !json.Valid(bytes.TrimSpace(stdout.Bytes())) { + t.Fatalf("stdout = %q, want completed import JSON", stdout.String()) + } +} + +func TestImportMediaResumesProcessingCheckpointWithoutUploading(t *testing.T) { + opts := testMediaResolutionOptions(t) + item := opts.Media[0] + checkpoint := &mediaCheckpoint{ + Schema: mediaCheckpointSchema, + Source: opts.Plan.Source, + Target: opts.Target, + BundleDirs: []string{opts.BundleDir}, + Entries: []mediaCheckpointEntry{{ + LogicalID: item.LogicalID, MediaType: item.MediaType, SHA256: item.SHA256, + Status: mediaStatusProcessing, AssetID: "asset-processing", PippitAssetID: "pippit-processing", + }}, } - media.uploadState = canvascore.StateReady - stdout.Reset() - stderr.Reset() - second := newImportCommand(&stdout, &stderr, deps) - second.SetArgs([]string{"--from", "libtv", "--url", testLibTVURL}) - if err := second.Execute(); err != nil { - t.Fatalf("second Execute() error = %v, stderr = %s", err, stderr.String()) + if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { + t.Fatal(err) } - if media.uploads != 1 || media.queries != 1 || executor.calls != 1 { - t.Fatalf("upload/query/execute = %d/%d/%d, want 1/1/1", media.uploads, media.queries, executor.calls) + api := &fakeImportMediaAPI{queryReady: []bool{false, true}} + var stderr bytes.Buffer + resolved, err := resolveImportMedia(context.Background(), opts, api, &stderr) + if err != nil { + t.Fatalf("resolveImportMedia() error = %v, stderr = %s", err, stderr.String()) } - if !strings.Contains(stderr.String(), `Media progress: processed=1/1 remaining=0 action=queried file="one.png"`) { - t.Fatalf("stderr = %q, want stable query progress", stderr.String()) + if api.uploads != 0 || api.queries != 2 || len(resolved.Media) != 1 { + t.Fatalf("uploads/queries/resolved = %d/%d/%#v, want 0/2/one", api.uploads, api.queries, resolved.Media) } - if !strings.Contains(stderr.String(), `Media progress: processed=0/1 remaining=1 action=checking file="one.png"`) { - t.Fatalf("stderr = %q, want pre-query progress before a potentially slow request", stderr.String()) + saved := readTestMediaCheckpoint(t, opts.CheckpointPath) + if len(saved.Entries) != 1 || saved.Entries[0].Status != mediaStatusReady { + t.Fatalf("checkpoint = %#v, want processing entry promoted to ready", saved) + } +} + +func TestImportMediaProcessingQueryAuthErrorStopsAndPreservesIDs(t *testing.T) { + opts := testMediaResolutionOptions(t) + item := opts.Media[0] + checkpoint := &mediaCheckpoint{ + Schema: mediaCheckpointSchema, + Source: opts.Plan.Source, + Target: opts.Target, + BundleDirs: []string{opts.BundleDir}, + Entries: []mediaCheckpointEntry{{ + LogicalID: item.LogicalID, MediaType: item.MediaType, SHA256: item.SHA256, + Status: mediaStatusProcessing, AssetID: "asset-durable", PippitAssetID: "pippit-durable", + }}, + } + if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { + t.Fatal(err) + } + api := &fakeImportMediaAPI{queryErr: errors.New("HTTP 401 Unauthorized")} + _, err := resolveImportMedia(context.Background(), opts, api, io.Discard) + if err == nil || !strings.Contains(err.Error(), "read/authentication error") || + !strings.Contains(err.Error(), "401 Unauthorized") || !strings.Contains(err.Error(), "durable IDs remain") { + t.Fatalf("resolveImportMedia() error = %v, want immediate explicit auth/query error", err) + } + if api.uploads != 0 || api.queries != 1 { + t.Fatalf("uploads/queries = %d/%d, want 0/1 without polling or re-upload", api.uploads, api.queries) + } + saved := readTestMediaCheckpoint(t, opts.CheckpointPath) + if len(saved.Entries) != 1 || saved.Entries[0].Status != mediaStatusProcessing || + saved.Entries[0].AssetID != "asset-durable" || saved.Entries[0].PippitAssetID != "pippit-durable" { + t.Fatalf("checkpoint = %#v, want processing status and durable IDs preserved", saved) + } +} + +func TestLibTVPNGAIGCFingerprintNormalizesOnlyVolatileIDs(t *testing.T) { + pixel := color.NRGBA{R: 20, G: 40, B: 60, A: 255} + first := testPNGWithITXt(t, pixel, testLibTVAIGCMetadata("libtv"+strings.Repeat("a", 32))) + second := testPNGWithITXt(t, pixel, testLibTVAIGCMetadata("libtv"+strings.Repeat("b", 32))) + firstRaw := sha256.Sum256(first) + secondRaw := sha256.Sum256(second) + if firstRaw == secondRaw { + t.Fatal("AIGC ID fixtures unexpectedly have the same raw SHA-256") + } + firstFingerprint, err := canonicalLibTVPNGFingerprint(first) + if err != nil { + t.Fatalf("canonicalLibTVPNGFingerprint(first) error = %v", err) + } + secondFingerprint, err := canonicalLibTVPNGFingerprint(second) + if err != nil { + t.Fatalf("canonicalLibTVPNGFingerprint(second) error = %v", err) + } + if firstFingerprint != secondFingerprint || !strings.HasPrefix(firstFingerprint, libTVPNGAIGCFingerprintPrefix) { + t.Fatalf("fingerprints = %q/%q, want identical versioned AIGC fingerprints", firstFingerprint, secondFingerprint) + } +} + +func TestLibTVPNGAIGCFingerprintPreservesAllOtherPNGContent(t *testing.T) { + pixel := color.NRGBA{R: 20, G: 40, B: 60, A: 255} + metadata := testLibTVAIGCMetadata("libtv" + strings.Repeat("a", 32)) + baseline := testPNGWithITXt(t, pixel, metadata) + baselineFingerprint, err := canonicalLibTVPNGFingerprint(baseline) + if err != nil { + t.Fatal(err) + } + idatChanged := testRewriteFirstPNGChunk(t, baseline, "IDAT", func(data []byte) { + data[0] ^= 0x01 + }) + apngOne := testInsertPNGChunkBeforeIEND(t, baseline, "acTL", []byte{0, 0, 0, 1, 0, 0, 0, 0}) + apngTwo := testInsertPNGChunkBeforeIEND(t, baseline, "acTL", []byte{0, 0, 0, 1, 0, 0, 0, 1}) + cases := map[string][]byte{ + "stable AIGC field": testPNGWithITXt( + t, pixel, strings.Replace(metadata, `"Label":"1"`, `"Label":"2"`, 1), + ), + "IDAT data": idatChanged, + "APNG chunk": apngOne, + "pixel data": testPNGWithITXt( + t, color.NRGBA{R: 21, G: 40, B: 60, A: 255}, + metadata, + ), + } + for name, payload := range cases { + t.Run(name, func(t *testing.T) { + fingerprint, err := canonicalLibTVPNGFingerprint(payload) + if err != nil { + t.Fatalf("canonicalLibTVPNGFingerprint() error = %v", err) + } + if fingerprint == baselineFingerprint { + t.Fatalf("fingerprint = %q, want change to affect canonical identity", fingerprint) + } + }) + } + apngOneFingerprint, err := canonicalLibTVPNGFingerprint(apngOne) + if err != nil { + t.Fatal(err) + } + apngTwoFingerprint, err := canonicalLibTVPNGFingerprint(apngTwo) + if err != nil { + t.Fatal(err) + } + if apngOneFingerprint == apngTwoFingerprint { + t.Fatal("changing APNG chunk data must change the canonical fingerprint") + } +} + +func TestLibTVPNGAIGCFingerprintRejectsInvalidStructure(t *testing.T) { + payload := testPNGWithITXt( + t, + color.NRGBA{R: 20, G: 40, B: 60, A: 255}, + testLibTVAIGCMetadata("libtv"+strings.Repeat("a", 32)), + ) + badCRC := append([]byte(nil), payload...) + badCRC[len(badCRC)-1] ^= 0x01 + for name, invalid := range map[string][]byte{ + "bad CRC": badCRC, + "trailing": append(append([]byte(nil), payload...), 0), + } { + t.Run(name, func(t *testing.T) { + if _, err := canonicalLibTVPNGFingerprint(invalid); err == nil { + t.Fatal("canonicalLibTVPNGFingerprint() error = nil, want structural rejection") + } + }) + } +} + +func TestImportMediaMigratesLegacyPNGCheckpointWhenOnlyITXtChanges(t *testing.T) { + oldPNG := testPNGWithITXt(t, color.NRGBA{R: 20, G: 40, B: 60, A: 255}, testLibTVAIGCMetadata("libtv"+strings.Repeat("a", 32))) + currentPNG := testPNGWithITXt(t, color.NRGBA{R: 20, G: 40, B: 60, A: 255}, testLibTVAIGCMetadata("libtv"+strings.Repeat("b", 32))) + if len(oldPNG) != len(currentPNG) { + t.Fatalf("PNG sizes = %d/%d, want metadata-only fixtures with equal size", len(oldPNG), len(currentPNG)) + } + opts, oldSHA := testPNGCheckpointMigrationOptions(t, oldPNG, currentPNG) + if oldSHA == opts.Media[0].SHA256 { + t.Fatal("metadata-only PNG fixtures unexpectedly have the same raw SHA-256") + } + oldPath := filepath.Join(opts.BundleRoot, "export-old", "media", "one.png") + oldFingerprint, err := importMediaContentFingerprint(oldPath, oldSHA) + if err != nil || oldFingerprint == "" || oldFingerprint != opts.Media[0].ContentFingerprint { + t.Fatalf("old/current fingerprints = %q/%q, error=%v, want equal normalized AIGC content", oldFingerprint, opts.Media[0].ContentFingerprint, err) + } + api := &fakeImportMediaAPI{} + resolved, err := resolveImportMedia(context.Background(), opts, api, io.Discard) + if err != nil { + t.Fatalf("resolveImportMedia() error = %v", err) + } + if api.uploads != 0 || api.queries != 1 || len(resolved.Media) != 1 || + resolved.Media[0].PippitAssetID != "pippit-legacy" { + t.Fatalf("uploads/queries/resolved = %d/%d/%#v, want durable legacy reuse", api.uploads, api.queries, resolved.Media) + } + saved := readTestMediaCheckpoint(t, opts.CheckpointPath) + if len(saved.Entries) != 1 || saved.Entries[0].SHA256 != oldSHA || + saved.Entries[0].ContentFingerprint != opts.Media[0].ContentFingerprint || + saved.Entries[0].CanonicalByteSize != int64(len(oldPNG)) { + t.Fatalf("checkpoint = %#v, want original uploaded identity plus normalized AIGC fingerprint", saved) + } + canonicalPlan, err := canonicalizeImportPlanMedia(opts.Plan, opts.Target, opts.CanvasJournalPath, opts.CheckpointPath) + if err != nil { + t.Fatalf("canonicalizeImportPlanMedia() error = %v", err) + } + if canonicalPlan.RequiredMedia[0].SHA256 != oldSHA || canonicalPlan.RequiredMedia[0].Metadata.ByteSize == nil || + *canonicalPlan.RequiredMedia[0].Metadata.ByteSize != int64(len(oldPNG)) { + t.Fatalf("canonical media = %#v, want original uploaded SHA/size", canonicalPlan.RequiredMedia[0]) + } +} + +func TestImportMediaRejectsLegacyPNGCheckpointWhenPixelsChange(t *testing.T) { + oldPNG := testPNGWithITXt(t, color.NRGBA{R: 20, G: 40, B: 60, A: 255}, testLibTVAIGCMetadata("libtv"+strings.Repeat("a", 32))) + currentPNG := testPNGWithITXt(t, color.NRGBA{R: 21, G: 40, B: 60, A: 255}, testLibTVAIGCMetadata("libtv"+strings.Repeat("b", 32))) + opts, oldSHA := testPNGCheckpointMigrationOptions(t, oldPNG, currentPNG) + api := &fakeImportMediaAPI{} + _, err := resolveImportMedia(context.Background(), opts, api, io.Discard) + if err == nil || !strings.Contains(err.Error(), "image content changed") { + t.Fatalf("resolveImportMedia() error = %v, want normalized-content mismatch rejection", err) + } + if api.uploads != 0 || api.queries != 0 { + t.Fatalf("uploads/queries = %d/%d, want no remote calls after pixel mismatch", api.uploads, api.queries) + } + saved := readTestMediaCheckpoint(t, opts.CheckpointPath) + if len(saved.Entries) != 1 || saved.Entries[0].SHA256 != oldSHA || saved.Entries[0].ContentFingerprint != "" { + t.Fatalf("checkpoint = %#v, want legacy entry left unchanged", saved) } } @@ -435,6 +802,8 @@ func TestImportMediaProgressReportsEmptySet(t *testing.T) { BundleRoot: bundleRoot, CanvasJournalPath: filepath.Join(root, "state", "canvas.journal.json"), CheckpointPath: filepath.Join(root, "state", "canvas.journal.json.media.json"), + PollInterval: time.Millisecond, + WaitTimeout: time.Second, } if err := os.MkdirAll(opts.BundleDir, 0o700); err != nil { t.Fatal(err) @@ -692,7 +1061,39 @@ func testImportDependencies( userConfigDir: func() (string, error) { return filepath.Join(root, "config"), nil }, target: func() string { return "https://xyq.jianying.com|ppe_cli_canvas_ak" }, authScope: func() string { return strings.Repeat("a", 64) }, + mediaPoll: time.Millisecond, + mediaTimeout: time.Second, + } +} + +func prepareSourceBoundResumeFixture( + t *testing.T, + root string, +) (canvasplan.Plan, *fakeImportExporter, string) { + t.Helper() + plan, _ := testImportPlan(t, false) + plan.RequiredMedia = nil + plan.Nodes = []canvasplan.Node{{ + LogicalID: "node:placeholder", SourceNodeID: "placeholder", Title: "Pending image", + Position: canvasplan.Position{X: 0, Y: 0}, Size: canvasplan.Size{Width: 100, Height: 100}, + Kind: "image-placeholder", TargetType: "biz/image", + }} + plan.Degradations = []json.RawMessage{json.RawMessage(`{"code":"test.placeholder"}`)} + exporter := &fakeImportExporter{plan: plan, mediaBytes: map[string][]byte{}} + journalPath := filepath.Join(root, "existing.journal.json") + if err := os.WriteFile(journalPath, []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + checkpoint := &mediaCheckpoint{ + Schema: mediaCheckpointSchema, + Source: plan.Source, + Target: "https://xyq.jianying.com|ppe_cli_canvas_ak", + Entries: []mediaCheckpointEntry{}, } + if err := saveMediaCheckpoint(journalPath+".media.json", checkpoint); err != nil { + t.Fatal(err) + } + return plan, exporter, journalPath } func testImportPlan(t *testing.T, degraded bool) (canvasplan.Plan, map[string][]byte) { @@ -752,7 +1153,140 @@ func testMediaResolutionOptions(t *testing.T) mediaResolutionOptions { BundleRoot: bundleRoot, CanvasJournalPath: filepath.Join(root, "state", "canvas.journal.json"), CheckpointPath: filepath.Join(root, "state", "canvas.journal.json.media.json"), + PollInterval: time.Millisecond, + WaitTimeout: time.Second, + } +} + +func testPNGCheckpointMigrationOptions(t *testing.T, oldPNG, currentPNG []byte) (mediaResolutionOptions, string) { + t.Helper() + root := t.TempDir() + bundleRoot := filepath.Join(root, "cache", "exports") + oldBundle := filepath.Join(bundleRoot, "export-old") + currentBundle := filepath.Join(bundleRoot, "export-current") + oldPlan, oldMedia := testSingleMediaPlan(t) + currentPlan := oldPlan + currentPlan.RequiredMedia = append([]canvasplan.MediaRequirement(nil), oldPlan.RequiredMedia...) + oldSize := int64(len(oldPNG)) + oldDigest := sha256.Sum256(oldPNG) + oldSHA := hex.EncodeToString(oldDigest[:]) + oldPlan.RequiredMedia[0].SHA256 = oldSHA + oldPlan.RequiredMedia[0].Metadata.ByteSize = &oldSize + oldMedia["media/one.png"] = oldPNG + currentSize := int64(len(currentPNG)) + currentDigest := sha256.Sum256(currentPNG) + currentPlan.RequiredMedia[0].SHA256 = hex.EncodeToString(currentDigest[:]) + currentPlan.RequiredMedia[0].Metadata.ByteSize = ¤tSize + currentMedia := map[string][]byte{"media/one.png": currentPNG} + oldExporter := &fakeImportExporter{plan: oldPlan, mediaBytes: oldMedia} + if _, err := oldExporter.Export(context.Background(), testLibTVURL, oldBundle, io.Discard); err != nil { + t.Fatalf("prepare old PNG export: %v", err) + } + currentExporter := &fakeImportExporter{plan: currentPlan, mediaBytes: currentMedia} + if _, err := currentExporter.Export(context.Background(), testLibTVURL, currentBundle, io.Discard); err != nil { + t.Fatalf("prepare current PNG export: %v", err) + } + media, err := readAndValidateExportMedia(currentBundle, currentPlan) + if err != nil { + t.Fatalf("validate current PNG export: %v", err) + } + opts := mediaResolutionOptions{ + Plan: currentPlan, + Media: media, + Target: "https://xyq.jianying.com|ppe_cli_canvas_ak", + BundleDir: currentBundle, + BundleRoot: bundleRoot, + CanvasJournalPath: filepath.Join(root, "state", "canvas.journal.json"), + CheckpointPath: filepath.Join(root, "state", "canvas.journal.json.media.json"), + PollInterval: time.Millisecond, + WaitTimeout: time.Second, } + checkpoint := &mediaCheckpoint{ + Schema: mediaCheckpointSchema, + Source: currentPlan.Source, + Target: opts.Target, + BundleDirs: []string{oldBundle}, + Entries: []mediaCheckpointEntry{{ + LogicalID: "media:image-1", MediaType: "image", SHA256: oldSHA, + Status: mediaStatusReady, AssetID: "asset-legacy", PippitAssetID: "pippit-legacy", + }}, + } + if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { + t.Fatalf("save legacy PNG checkpoint: %v", err) + } + return opts, oldSHA +} + +func testPNGWithITXt(t *testing.T, pixel color.NRGBA, metadata string) []byte { + t.Helper() + imageValue := image.NewNRGBA(image.Rect(0, 0, 2, 2)) + for y := 0; y < 2; y++ { + for x := 0; x < 2; x++ { + imageValue.SetNRGBA(x, y, pixel) + } + } + var encoded bytes.Buffer + if err := png.Encode(&encoded, imageValue); err != nil { + t.Fatalf("encode PNG fixture: %v", err) + } + data := append([]byte("AIGC\x00\x00\x00\x00\x00"), []byte(metadata)...) + return testInsertPNGChunkBeforeIEND(t, encoded.Bytes(), "iTXt", data) +} + +func testLibTVAIGCMetadata(id string) string { + return fmt.Sprintf( + `{"Label":"1","ContentProducer":%q,"ProduceID":%q,"ReservedCode1":"","ContentPropagator":%q,"PropagateID":%q,"ReservedCode2":""}`, + libTVAIGCProducer, + id, + libTVAIGCProducer, + id, + ) +} + +func testInsertPNGChunkBeforeIEND(t *testing.T, payload []byte, chunkType string, data []byte) []byte { + t.Helper() + if len(chunkType) != 4 { + t.Fatalf("PNG chunk type %q must have four bytes", chunkType) + } + if len(payload) < 12 || string(payload[len(payload)-8:len(payload)-4]) != "IEND" { + t.Fatal("PNG fixture has no terminal IEND chunk") + } + chunk := make([]byte, 12+len(data)) + binary.BigEndian.PutUint32(chunk[:4], uint32(len(data))) + copy(chunk[4:8], chunkType) + copy(chunk[8:8+len(data)], data) + checksumInput := append([]byte(chunkType), data...) + binary.BigEndian.PutUint32(chunk[8+len(data):], crc32.ChecksumIEEE(checksumInput)) + result := make([]byte, 0, len(payload)+len(chunk)) + result = append(result, payload[:len(payload)-12]...) + result = append(result, chunk...) + result = append(result, payload[len(payload)-12:]...) + return result +} + +func testRewriteFirstPNGChunk(t *testing.T, payload []byte, expectedType string, rewrite func([]byte)) []byte { + t.Helper() + result := append([]byte(nil), payload...) + for offset := len(pngFileSignature); offset+12 <= len(result); { + length := int(binary.BigEndian.Uint32(result[offset : offset+4])) + end := offset + 12 + length + if end > len(result) { + t.Fatal("PNG fixture has a truncated chunk") + } + if string(result[offset+4:offset+8]) == expectedType { + data := result[offset+8 : offset+8+length] + if len(data) == 0 { + t.Fatalf("PNG %s fixture chunk has no data", expectedType) + } + rewrite(data) + checksumInput := append([]byte(expectedType), data...) + binary.BigEndian.PutUint32(result[offset+8+length:end], crc32.ChecksumIEEE(checksumInput)) + return result + } + offset = end + } + t.Fatalf("PNG fixture has no %s chunk", expectedType) + return nil } func readTestMediaCheckpoint(t *testing.T, path string) mediaCheckpoint { From 5d88edd296d316a81aff0a495faba61109dd94c1 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 19:24:37 +0800 Subject: [PATCH 29/48] feat(canvas): add interactive import TUI Co-authored-by: Codex <codex@openai.com> --- README.md | 2 +- cmd/canvas/import.go | 2 +- cmd/canvas/import_prompt.go | 43 +++++++++++- cmd/canvas/import_prompt_tui.go | 97 ++++++++++++++++++++++++++++ cmd/canvas/import_prompt_tui_test.go | 96 +++++++++++++++++++++++++++ go.mod | 30 ++++++++- go.sum | 77 +++++++++++++++++++++- 7 files changed, 340 insertions(+), 7 deletions(-) create mode 100644 cmd/canvas/import_prompt_tui.go create mode 100644 cmd/canvas/import_prompt_tui_test.go diff --git a/README.md b/README.md index 344b898..d002217 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ LibTV 迁移只是上述通用画布能力的 CLI 编排层，服务端不识别 pippit-tool-cli --ppe-env ppe_cli_canvas_ak canvas import ``` -CLI 会用编号选项逐步询问来源、journal 策略与是否打开结果，只有每个项目唯一的 LibTV 链接需要粘贴。journal 选择 Automatic 即使用权限受控的自动路径，不需要设置环境变量。交互式导入遇到已知的非致命降级时会输出 warning 后自动继续，并在最终 JSON 中保留 `degradation_count`。源端节点处理、素材下载与 Pippit 素材上传会在 stderr 显示已处理/总数/剩余数，画布创建、写入和回读校验会显示当前阶段；最终 stdout 仍只输出一行 JSON。 +CLI 会在交互终端中显示彩色向导：使用 ↑/↓ 移动、Enter 确认，逐步选择来源、journal 策略与是否打开结果，只有每个项目唯一的 LibTV 链接需要粘贴。journal 选择 Automatic 即使用权限受控的自动路径，不需要设置环境变量。交互式导入遇到已知的非致命降级时会输出 warning 后自动继续，并在最终 JSON 中保留 `degradation_count`。源端节点处理、素材下载与 Pippit 素材上传会在 stderr 显示已处理/总数/剩余数，画布创建、写入和回读校验会显示当前阶段；最终 stdout 仍只输出一行 JSON。设置 `PIPPIT_CLI_ACCESSIBLE=1` 可切换为无控制序列的朴素提示模式。 供 Agent、CI 或其它非交互场景使用时，仍可显式传入 `--from`、`--url`、`--accept-degradations` 和 `--open`；`--journal` 始终可选，省略时使用自动路径。 diff --git a/cmd/canvas/import.go b/cmd/canvas/import.go index 352f937..f49b814 100644 --- a/cmd/canvas/import.go +++ b/cmd/canvas/import.go @@ -115,7 +115,7 @@ func newImportCommand( opts.JournalExplicit = cmd.Flags().Changed("journal") opts.AcceptDegradationsExplicit = cmd.Flags().Changed("accept-degradations") prepared, prompts, err := prepareCanvasImportOptions( - cmd.InOrStdin(), opts, dependencies.isInteractive, stderr, + cmd.Context(), cmd.InOrStdin(), opts, dependencies.isInteractive, stderr, ) if err != nil { return err diff --git a/cmd/canvas/import_prompt.go b/cmd/canvas/import_prompt.go index 880480a..8aaeff7 100644 --- a/cmd/canvas/import_prompt.go +++ b/cmd/canvas/import_prompt.go @@ -2,6 +2,7 @@ package canvas import ( "bufio" + "context" "fmt" "io" "os" @@ -14,6 +15,7 @@ type importPromptSession struct { reader *bufio.Reader stderr io.Writer eof bool + tui *importPromptTUI } type importPromptChoice struct { @@ -21,6 +23,37 @@ type importPromptChoice struct { aliases []string } +func newImportPromptSession(ctx context.Context, input io.Reader, stderr io.Writer) *importPromptSession { + return newImportPromptSessionWithTUI( + ctx, + input, + stderr, + importInputIsInteractive(input) && importOutputIsInteractive(stderr) && + os.Getenv("PIPPIT_CLI_ACCESSIBLE") == "", + ) +} + +func importOutputIsInteractive(output io.Writer) bool { + file, ok := output.(*os.File) + return ok && importFileIsTerminal(file) +} + +func newImportPromptSessionWithTUI( + ctx context.Context, + input io.Reader, + stderr io.Writer, + enableTUI bool, +) *importPromptSession { + session := &importPromptSession{ + reader: bufio.NewReader(input), + stderr: stderr, + } + if enableTUI { + session.tui = &importPromptTUI{ctx: ctx, input: input, output: stderr} + } + return session +} + func importInputIsInteractive(input io.Reader) bool { file, ok := input.(*os.File) if !ok { @@ -30,6 +63,7 @@ func importInputIsInteractive(input io.Reader) bool { } func prepareCanvasImportOptions( + ctx context.Context, input io.Reader, opts importOptions, isInteractive func(io.Reader) bool, @@ -45,7 +79,7 @@ func prepareCanvasImportOptions( importFlagsHint, ) } - prompts := &importPromptSession{reader: bufio.NewReader(input), stderr: stderr} + prompts := newImportPromptSession(ctx, input, stderr) if strings.TrimSpace(opts.Provider) == "" { _, err := prompts.askChoice( "Source provider:", @@ -130,6 +164,9 @@ func (prompts *importPromptSession) askChoice( choices []importPromptChoice, defaultChoice int, ) (int, error) { + if prompts.tui != nil { + return prompts.tui.askChoice(title, choices, defaultChoice) + } for { fmt.Fprintln(prompts.stderr, title) for index, choice := range choices { @@ -165,6 +202,10 @@ func containsImportPromptAlias(values []string, expected string) bool { } func (prompts *importPromptSession) readLine(label string) (string, bool, error) { + if prompts.tui != nil { + value, err := prompts.tui.readLine(label) + return value, false, err + } fmt.Fprint(prompts.stderr, label) if prompts.eof { return "", true, nil diff --git a/cmd/canvas/import_prompt_tui.go b/cmd/canvas/import_prompt_tui.go new file mode 100644 index 0000000..d1fc0e0 --- /dev/null +++ b/cmd/canvas/import_prompt_tui.go @@ -0,0 +1,97 @@ +package canvas + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "github.com/charmbracelet/huh" +) + +const importPromptWidth = 72 + +var errCanvasImportSetupCanceled = errors.New("Canvas import setup canceled") + +type importPromptTUI struct { + ctx context.Context + input io.Reader + output io.Writer +} + +func (prompt *importPromptTUI) askChoice( + title string, + choices []importPromptChoice, + defaultChoice int, +) (int, error) { + if defaultChoice < 1 || defaultChoice > len(choices) { + return 0, fmt.Errorf("invalid default Canvas import choice %d", defaultChoice) + } + selected := defaultChoice + options := make([]huh.Option[int], 0, len(choices)) + for index, choice := range choices { + options = append(options, huh.NewOption(choice.label, index+1)) + } + field := huh.NewSelect[int](). + Title(strings.TrimSpace(title)). + Description("Use ↑/↓ to move, Enter to select"). + Options(options...). + Value(&selected) + if err := prompt.run(field); err != nil { + return 0, err + } + return selected, nil +} + +func (prompt *importPromptTUI) readLine(label string) (string, error) { + value := "" + title := strings.TrimSpace(strings.TrimSuffix(label, ": ")) + field := huh.NewInput(). + Title(title). + Description("Paste a value, then press Enter"). + Prompt("› "). + Value(&value) + switch title { + case "LibTV canvas URL": + field.Validate(func(value string) error { + if _, err := normalizeLibTVURL(value); err != nil { + return err + } + return nil + }) + case "Custom journal path": + field.Validate(func(value string) error { + if strings.TrimSpace(value) == "" { + return fmt.Errorf("a custom journal path is required") + } + return nil + }) + } + if err := prompt.run(field); err != nil { + return "", err + } + return strings.TrimSpace(value), nil +} + +func (prompt *importPromptTUI) run(field huh.Field) error { + form := huh.NewForm(huh.NewGroup(field)). + WithTheme(huh.ThemeCharm()). + WithWidth(importPromptWidth). + WithInput(prompt.input). + WithOutput(prompt.output) + ctx := prompt.ctx + if ctx == nil { + ctx = context.Background() + } + if err := form.RunWithContext(ctx); err != nil { + if ctx.Err() != nil { + return fmt.Errorf("%w: %w", errCanvasImportSetupCanceled, ctx.Err()) + } + if errors.Is(err, huh.ErrUserAborted) { + return errCanvasImportSetupCanceled + } + return fmt.Errorf("run Canvas import terminal prompt: %w", err) + } + return nil +} diff --git a/cmd/canvas/import_prompt_tui_test.go b/cmd/canvas/import_prompt_tui_test.go new file mode 100644 index 0000000..d77e18c --- /dev/null +++ b/cmd/canvas/import_prompt_tui_test.go @@ -0,0 +1,96 @@ +package canvas + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "sync" + "testing" + "time" +) + +type delayedImportPromptReader struct { + io.Reader + once sync.Once +} + +func (reader *delayedImportPromptReader) Read(buffer []byte) (int, error) { + reader.once.Do(func() { time.Sleep(25 * time.Millisecond) }) + return reader.Reader.Read(buffer) +} + +func TestImportPromptTUISelectUsesArrowKeys(t *testing.T) { + t.Setenv("TERM", "xterm-256color") + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + var output bytes.Buffer + session := newImportPromptSessionWithTUI( + ctx, + &delayedImportPromptReader{Reader: bytes.NewBufferString("\x1b[B\r")}, + &output, + true, + ) + selected, err := session.askChoice( + "Resume journal:", + []importPromptChoice{ + {label: "Automatic (recommended)"}, + {label: "Custom path"}, + }, + 1, + ) + if err != nil { + t.Fatalf("askChoice() error = %v; output = %q", err, output.String()) + } + if selected != 2 { + t.Fatalf("askChoice() = %d, want arrow-down selection 2", selected) + } + for _, expected := range []string{"Resume journal", "Automatic", "Custom path"} { + if !strings.Contains(output.String(), expected) { + t.Fatalf("TUI output missing %q: %q", expected, output.String()) + } + } +} + +func TestImportPromptTUIReadsPastedURL(t *testing.T) { + t.Setenv("TERM", "xterm-256color") + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + var output bytes.Buffer + session := newImportPromptSessionWithTUI( + ctx, + &delayedImportPromptReader{Reader: bytes.NewBufferString(testLibTVURL + "\r")}, + &output, + true, + ) + value, eof, err := session.readLine("LibTV canvas URL: ") + if err != nil { + t.Fatalf("readLine() error = %v; output = %q", err, output.String()) + } + if eof { + t.Fatal("TUI readLine() unexpectedly reported EOF") + } + if value != testLibTVURL { + t.Fatalf("readLine() = %q, want pasted URL", value) + } + if !strings.Contains(output.String(), "LibTV canvas URL") { + t.Fatalf("TUI output = %q, want input title", output.String()) + } +} + +func TestImportPromptTUIHonorsContextCancellation(t *testing.T) { + t.Setenv("TERM", "xterm-256color") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var output bytes.Buffer + session := newImportPromptSessionWithTUI(ctx, bytes.NewBuffer(nil), &output, true) + _, err := session.askChoice( + "After import:", + []importPromptChoice{{label: "Open Canvas"}, {label: "Do not open"}}, + 1, + ) + if err == nil || !errors.Is(err, errCanvasImportSetupCanceled) { + t.Fatalf("askChoice() error = %v, want actionable cancellation", err) + } +} diff --git a/go.mod b/go.mod index fe5d4d5..0e0474b 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,47 @@ module github.com/Pippit-dev/pippit-cli -go 1.23 +go 1.23.0 require ( github.com/bytedance/sonic v1.15.1 + github.com/charmbracelet/huh v1.0.0 github.com/spf13/cobra v1.8.1 - golang.org/x/sys v0.27.0 + golang.org/x/sys v0.33.0 ) require ( + github.com/atotto/clipboard v0.1.4 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/catppuccin/go v0.3.0 // indirect + github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect + github.com/charmbracelet/bubbletea v1.3.6 // indirect + github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.9.3 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13 // indirect + github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect github.com/cloudwego/base64x v0.1.6 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.11.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect + golang.org/x/sync v0.15.0 // indirect + golang.org/x/text v0.23.0 // indirect ) diff --git a/go.sum b/go.sum index 607aa5b..e145c77 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,84 @@ +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= +github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.1 h1:nJD5PmM0vY7J8CT6MxoqbVAAMhkSmV2HgRAUrrpLoOw= github.com/bytedance/sonic v1.15.1/go.mod h1:mT2NbXunuaEbnZ+mRIX/vYqKISmgEuHFDI4UzmKx2SA= github.com/bytedance/sonic/loader v0.5.1 h1:Ygpfa9zwRCCKSlrp5bBP/b/Xzc3VxsAW+5NIYXrOOpI= github.com/bytedance/sonic/loader v0.5.1/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/catppuccin/go v0.3.0 h1:d+0/YicIq+hSTo5oPuRi5kOpqkVA5tAsU6dNhvRu+aY= +github.com/catppuccin/go v0.3.0/go.mod h1:8IHJuMGaUUjQM82qBrGNBv7LFq6JI3NnQCF6MOlZjpc= +github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 h1:JFgG/xnwFfbezlUnFMJy0nusZvytYysV4SCS2cYbvws= +github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7/go.mod h1:ISC1gtLcVilLOf23wvTfoQuYbW2q0JevFxPfUzZ9Ybw= +github.com/charmbracelet/bubbletea v1.3.6 h1:VkHIxPJQeDt0aFJIsVxw8BQdh/F/L2KKZGsK6et5taU= +github.com/charmbracelet/bubbletea v1.3.6/go.mod h1:oQD9VCRQFF8KplacJLo28/jofOI2ToOfGYeFgBBxHOc= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= +github.com/charmbracelet/huh v1.0.0 h1:wOnedH8G4qzJbmhftTqrpppyqHakl/zbbNdXIWJyIxw= +github.com/charmbracelet/huh v1.0.0/go.mod h1:5YVc+SlZ1IhQALxRPpkGwwEKftN/+OlJlnJYlDRFqN4= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.9.3 h1:BXt5DHS/MKF+LjuK4huWrC6NCvHtexww7dMayh6GXd0= +github.com/charmbracelet/x/ansi v0.9.3/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= +github.com/charmbracelet/x/cellbuf v0.0.13 h1:/KBBKHuVRbq1lYx5BzEHBAFBP8VcQzJejZ/IA3iR28k= +github.com/charmbracelet/x/cellbuf v0.0.13/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/conpty v0.1.0 h1:4zc8KaIcbiL4mghEON8D72agYtSeIgq8FSThSPQIb+U= +github.com/charmbracelet/x/conpty v0.1.0/go.mod h1:rMFsDJoDwVmiYM10aD4bH2XiRgwI7NYJtQgl5yskjEQ= +github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86 h1:JSt3B+U9iqk37QUU2Rvb6DSBYRLtWqFqfxf8l5hOZUA= +github.com/charmbracelet/x/errors v0.0.0-20240508181413-e8d8b6e2de86/go.mod h1:2P0UgXMEa6TsToMSuFqKFQR+fZTO9CNGUNokkPatT/0= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ= +github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 h1:qko3AQ4gK1MTS/de7F5hPGx6/k1u0w4TeYmBFwzYVP4= +github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0/go.mod h1:pBhA0ybfXv6hDjQUZ7hk1lVxBiUbupdw5R31yPUViVQ= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= +github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= +github.com/charmbracelet/x/xpty v0.1.2 h1:Pqmu4TEJ8KeA9uSkISKMU3f+C1F6OGBn8ABuGlqCbtI= +github.com/charmbracelet/x/xpty v0.1.2/go.mod h1:XK2Z0id5rtLWcpeNiMYBccNNBrP2IJnzHI0Lq13Xzq4= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -33,10 +96,20 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 8afa8831e0f3e14b7b9318c188e8281fa652a179 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 22:47:18 +0800 Subject: [PATCH 30/48] feat(auth): support runtime access key updates Co-authored-by: Codex <codex@openai.com> --- cmd/root.go | 18 +++++-- cmd/root_test.go | 31 ++++++++++++ internal/common/access_key.go | 27 ++++++++--- internal/common/access_key_test.go | 77 ++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 10 deletions(-) create mode 100644 internal/common/access_key_test.go diff --git a/cmd/root.go b/cmd/root.go index 1c4f654..47365cb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,14 +26,24 @@ func Execute() error { func NewRootCommand(stdout, stderr io.Writer) *cobra.Command { cfg := config.Load() - client := common.NewHTTPClientWithPPEEnv( + runner := newRootRunner(cfg) + return newRootCommand(stdout, stderr, runner) +} + +func newRootRunner(cfg *config.Config) *common.Runner { + runner := common.NewRunner(cfg, nil) + runner.Client = common.NewHTTPClientWithPPEEnv( cfg.BaseURL, cfg.HTTPTimeout, - common.NewAccessKeyAuthorizer(cfg.AccessKey), + common.NewAccessKeyProviderAuthorizer(func() string { + if runner.Config == nil { + return "" + } + return runner.Config.AccessKey + }), func() string { return cfg.PPEEnv }, ) - runner := common.NewRunner(cfg, client) - return newRootCommand(stdout, stderr, runner) + return runner } func newRootCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { diff --git a/cmd/root_test.go b/cmd/root_test.go index daf6c04..4ba96bc 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -2,14 +2,45 @@ package cmd import ( "bytes" + "context" + "net/http" + "net/http/httptest" "strings" "testing" + "time" "github.com/Pippit-dev/pippit-cli/internal/common" "github.com/Pippit-dev/pippit-cli/internal/config" "github.com/spf13/cobra" ) +func TestRootRunnerReadsUpdatedAccessKeyForEveryRequest(t *testing.T) { + received := make([]string, 0, 2) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + received = append(received, request.Header.Get("Authorization")) + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"ok":true}`)) + })) + defer server.Close() + + cfg := config.Load() + cfg.BaseURL = server.URL + cfg.HTTPTimeout = time.Second + cfg.AccessKey = "first-key" + runner := newRootRunner(cfg) + + for _, accessKey := range []string{"first-key", "second-key"} { + runner.Config.AccessKey = accessKey + var response map[string]any + if err := runner.Client.SendRequest(context.Background(), "/probe", map[string]any{}, &response); err != nil { + t.Fatalf("SendRequest(%q) error = %v", accessKey, err) + } + } + if got, want := strings.Join(received, ","), "Bearer first-key,Bearer second-key"; got != want { + t.Fatalf("Authorization headers = %q, want %q", got, want) + } +} + func TestPPEEnvFlagOverridesEnvironment(t *testing.T) { t.Setenv(config.EnvPPEEnv, "ppe_from_env") cfg, root, ran := newPPEFlagTestRoot(t) diff --git a/internal/common/access_key.go b/internal/common/access_key.go index 8b4a337..042693a 100644 --- a/internal/common/access_key.go +++ b/internal/common/access_key.go @@ -10,22 +10,37 @@ import ( ) type accessKeyAuthorizer struct { - accessKey string + accessKey func() string } func NewAccessKeyAuthorizer(accessKey string) RequestAuthorizer { - return &accessKeyAuthorizer{accessKey: strings.TrimSpace(accessKey)} + trimmed := strings.TrimSpace(accessKey) + return NewAccessKeyProviderAuthorizer(func() string { return trimmed }) +} + +// NewAccessKeyProviderAuthorizer resolves the Access Key immediately before +// each request. This lets interactive commands update their in-memory runtime +// configuration without rebuilding the shared HTTP client. +func NewAccessKeyProviderAuthorizer(accessKey func() string) RequestAuthorizer { + if accessKey == nil { + accessKey = func() string { return "" } + } + return &accessKeyAuthorizer{accessKey: accessKey} } func (a *accessKeyAuthorizer) Inject(ctx context.Context, req *http.Request) error { if err := ctx.Err(); err != nil { return err } - if a.accessKey == "" && req.Method == http.MethodPost { - return fmt.Errorf("%s 缺失. 请前往小云雀官网个人设置页创建 Access Key，地址：https://xyq.jianying.com/home?tab_name=home\n配置后重试：\n export %s=\"<your-access-key>\"", config.EnvXYQAccessKey, config.EnvXYQAccessKey) + accessKey := "" + if a != nil && a.accessKey != nil { + accessKey = strings.TrimSpace(a.accessKey()) + } + if accessKey == "" && req.Method == http.MethodPost { + return fmt.Errorf("%s 缺失；请前往小云雀官网个人设置页创建 Access Key，地址：https://xyq.jianying.com/home?tab_name=home\n配置后重试：\n export %s=\"<your-access-key>\"", config.EnvXYQAccessKey, config.EnvXYQAccessKey) } - if a.accessKey != "" { - req.Header.Set("Authorization", "Bearer "+a.accessKey) + if accessKey != "" { + req.Header.Set("Authorization", "Bearer "+accessKey) } return nil } diff --git a/internal/common/access_key_test.go b/internal/common/access_key_test.go new file mode 100644 index 0000000..24a94b5 --- /dev/null +++ b/internal/common/access_key_test.go @@ -0,0 +1,77 @@ +package common + +import ( + "context" + "net/http" + "strings" + "testing" +) + +func TestAccessKeyProviderAuthorizerReadsLatestValue(t *testing.T) { + accessKey := " first-key " + authorizer := NewAccessKeyProviderAuthorizer(func() string { return accessKey }) + + first := newAccessKeyTestRequest(t, http.MethodPost) + if err := authorizer.Inject(context.Background(), first); err != nil { + t.Fatalf("Inject(first) error = %v", err) + } + if got := first.Header.Get("Authorization"); got != "Bearer first-key" { + t.Fatalf("first Authorization = %q, want latest first key", got) + } + + accessKey = "second-key" + second := newAccessKeyTestRequest(t, http.MethodPost) + if err := authorizer.Inject(context.Background(), second); err != nil { + t.Fatalf("Inject(second) error = %v", err) + } + if got := second.Header.Get("Authorization"); got != "Bearer second-key" { + t.Fatalf("second Authorization = %q, want updated key", got) + } +} + +func TestAccessKeyProviderAuthorizerRejectsMissingKeyWithoutLeakingPriorValue(t *testing.T) { + accessKey := "prior-secret" + authorizer := NewAccessKeyProviderAuthorizer(func() string { return accessKey }) + accessKey = " " + + err := authorizer.Inject(context.Background(), newAccessKeyTestRequest(t, http.MethodPost)) + if err == nil || !strings.Contains(err.Error(), "XYQ_ACCESS_KEY 缺失") { + t.Fatalf("Inject() error = %v, want missing Access Key guidance", err) + } + if strings.Contains(err.Error(), "prior-secret") { + t.Fatalf("Inject() error leaks a previous Access Key: %v", err) + } +} + +func TestAccessKeyProviderAuthorizerAllowsUnauthenticatedRead(t *testing.T) { + authorizer := NewAccessKeyProviderAuthorizer(nil) + request := newAccessKeyTestRequest(t, http.MethodGet) + + if err := authorizer.Inject(context.Background(), request); err != nil { + t.Fatalf("Inject() error = %v", err) + } + if got := request.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want empty", got) + } +} + +func TestAccessKeyAuthorizerKeepsConstantAPIBehavior(t *testing.T) { + authorizer := NewAccessKeyAuthorizer(" constant-key ") + request := newAccessKeyTestRequest(t, http.MethodPost) + + if err := authorizer.Inject(context.Background(), request); err != nil { + t.Fatalf("Inject() error = %v", err) + } + if got := request.Header.Get("Authorization"); got != "Bearer constant-key" { + t.Fatalf("Authorization = %q, want trimmed constant key", got) + } +} + +func newAccessKeyTestRequest(t *testing.T, method string) *http.Request { + t.Helper() + request, err := http.NewRequest(method, "https://example.test/api", nil) + if err != nil { + t.Fatalf("NewRequest() error = %v", err) + } + return request +} From f57a77af685cbad4a5ea8126ea71d254c1fd2087 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 22:47:38 +0800 Subject: [PATCH 31/48] feat(canvas): preflight LibTV authentication Co-authored-by: Codex <codex@openai.com> --- adapters/libtv/README.md | 15 +++ adapters/libtv/cli.mjs | 31 +++-- adapters/libtv/exporter.mjs | 132 +++++++++++++------- adapters/libtv/exporter.test.mjs | 141 +++++++++++++++++---- cmd/canvas/import_export.go | 104 +++++++++++++--- cmd/canvas/import_export_auth_test.go | 169 ++++++++++++++++++++++++++ 6 files changed, 503 insertions(+), 89 deletions(-) create mode 100644 cmd/canvas/import_export_auth_test.go diff --git a/adapters/libtv/README.md b/adapters/libtv/README.md index 0ce6c0e..5c6a95c 100644 --- a/adapters/libtv/README.md +++ b/adapters/libtv/README.md @@ -13,6 +13,21 @@ node adapters/libtv/cli.mjs export \ --output-dir ./libtv-bundle ``` +Before an import starts reading a project or downloading media, callers can +run the independent authentication preflight: + +```bash +node adapters/libtv/cli.mjs auth +``` + +This command only prepares the verified LibTV CLI and runs `account info`. If +the account is not authenticated, it runs `login web --open` and verifies the +account again. It never receives or reads a project URL, node, or media file. +Progress and browser-login output go to stderr; successful stdout remains one +JSON object using `pippit-libtv-auth-result/0.1`. `--non-interactive` reports +`AUTH_REQUIRED` without trying to read any project data, so an orchestration +layer can retry after arranging authentication. + An explicit `--libtv-cli`, `LIBTV_CLI_BINARY`, or `LIBTV_CLI_PATH` opts into a user-managed binary after a version check. Without an explicit override, the exporter never executes `libtv` from `PATH` or `~/.libtv`; it goes directly to diff --git a/adapters/libtv/cli.mjs b/adapters/libtv/cli.mjs index e3b461c..bc96d62 100755 --- a/adapters/libtv/cli.mjs +++ b/adapters/libtv/cli.mjs @@ -4,7 +4,7 @@ import { chmod, readFile, writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; -import { exportLibTVURL } from './exporter.mjs'; +import { exportLibTVURL, preflightLibTVAuth } from './exporter.mjs'; import { convertSnapshotToCanvasPlan } from './plan.mjs'; const BOOLEAN_FLAGS = new Set(['non-interactive']); @@ -20,7 +20,7 @@ function parseArgs(argv) { } const value = rest[index + 1]; if (!key?.startsWith('--') || !value || value.startsWith('--')) { - throw new Error(`invalid argument near ${key ?? '<end>'}`); + throw new Error(`参数格式无效，位置：${key ?? '参数末尾'}`); } args[key.slice(2)] = value; index += 1; @@ -41,9 +41,19 @@ async function runExport(args) { process.stdout.write(`${JSON.stringify(result)}\n`); } +async function runAuth(args) { + const result = await preflightLibTVAuth({ + binary: args['libtv-cli'], + nonInteractive: Boolean(args['non-interactive']), + env: process.env, + onProgress: (message) => process.stderr.write(`${message}\n`), + }); + process.stdout.write(`${JSON.stringify(result)}\n`); +} + function required(args, key) { const value = args[key]?.trim(); - if (!value) throw new Error(`--${key} is required`); + if (!value) throw new Error(`缺少必填参数 --${key}`); return value; } @@ -52,7 +62,7 @@ async function readJson(path) { try { return JSON.parse(text); } catch (error) { - throw new Error(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`); + throw new Error(`${path} 不是有效的 JSON：${error instanceof Error ? error.message : String(error)}`); } } @@ -91,12 +101,17 @@ async function main(argv = process.argv.slice(2)) { await runExport(args); return; } + if (args.command === 'auth') { + await runAuth(args); + return; + } throw new Error( - 'usage:\n' + - ' node adapters/libtv/cli.mjs export --url <LibTV canvas URL> --output-dir <new directory> ' + - '[--libtv-cli <path>] [--non-interactive] [--title <title>]\n' + + '用法：\n' + + ' node adapters/libtv/cli.mjs auth [--libtv-cli <路径>] [--non-interactive]\n' + + ' node adapters/libtv/cli.mjs export --url <LibTV 画布链接> --output-dir <新目录> ' + + '[--libtv-cli <路径>] [--non-interactive] [--title <标题>]\n' + ' node adapters/libtv/cli.mjs plan --snapshot <snapshot.json> ' + - '[--media-manifest <manifest.json>] [--title <title>] --output <plan.json|->', + '[--media-manifest <manifest.json>] [--title <标题>] --output <plan.json|->', ); } diff --git a/adapters/libtv/exporter.mjs b/adapters/libtv/exporter.mjs index 3974bc9..c7993d7 100644 --- a/adapters/libtv/exporter.mjs +++ b/adapters/libtv/exporter.mjs @@ -22,6 +22,7 @@ import { import { convertSnapshotToCanvasPlan } from './plan.mjs'; const EXPORT_RESULT_SCHEMA = 'pippit-libtv-export-result/0.1'; +const AUTH_RESULT_SCHEMA = 'pippit-libtv-auth-result/0.1'; const MEDIA_MANIFEST_SCHEMA = 'pippit-libtv-media-manifest/0.1'; const SUPPORTED_NODE_TYPES = new Set(['group', 'image', 'video', 'audio', 'video-clip']); const MEDIA_NODE_TYPES = new Set(['image', 'video', 'audio']); @@ -49,17 +50,17 @@ function parseLibTVCanvasURL(value) { try { url = new URL(String(value)); } catch { - throw new LibTVExportError('INVALID_URL', 'LibTV URL is invalid'); + throw new LibTVExportError('INVALID_URL', 'LibTV 画布链接无效'); } if (url.protocol !== 'https:' || !['www.liblib.tv', 'liblib.tv'].includes(url.hostname) || url.pathname !== '/canvas') { - throw new LibTVExportError('INVALID_URL', 'expected an HTTPS LibTV canvas URL'); + throw new LibTVExportError('INVALID_URL', '请输入 HTTPS 格式的 LibTV 画布链接'); } if (url.username || url.password || url.searchParams.getAll('projectId').length !== 1) { - throw new LibTVExportError('INVALID_URL', 'LibTV URL must contain exactly one projectId and no credentials'); + throw new LibTVExportError('INVALID_URL', 'LibTV 画布链接必须且只能包含一个 projectId，且不能包含账号凭据'); } const projectId = url.searchParams.get('projectId')?.trim() ?? ''; if (!/^(?:[0-9a-f]{32}|[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})$/i.test(projectId)) { - throw new LibTVExportError('INVALID_URL', 'LibTV projectId must be a 32-hex ID or UUID'); + throw new LibTVExportError('INVALID_URL', 'LibTV projectId 必须是 32 位十六进制 ID 或 UUID'); } return { projectId }; } @@ -162,7 +163,7 @@ async function locateLibTVCLI(options = {}) { const result = await runner.capture(['--version']); const version = result.exitCode === 0 ? result.stdout.trim().match(/\d+\.\d+\.\d+(?:[-+][\w.-]+)?/)?.[0] : undefined; if (version && supportsExporterVersion(version)) return { runner, version }; - throw new LibTVExportError('CLI_UNAVAILABLE', `configured LibTV CLI is unavailable or invalid: ${override}`); + throw new LibTVExportError('CLI_UNAVAILABLE', `配置的 LibTV 命令行工具不可用或版本无效：${override}`); } let bootstrapped; try { @@ -174,61 +175,99 @@ async function locateLibTVCLI(options = {}) { } catch (error) { throw new LibTVExportError( 'CLI_BOOTSTRAP_FAILED', - `LibTV CLI bootstrap failed: ${error instanceof Error ? error.message : String(error)}. ` + - `Official installer metadata: ${OFFICIAL_INSTALLERS.shell}`, + `准备 LibTV 命令行工具失败：${error instanceof Error ? error.message : String(error)}。` + + `官方安装信息：${OFFICIAL_INSTALLERS.shell}`, ); } const runner = createCommandRunner(bootstrapped, environment); const result = await runner.capture(['--version']); const version = result.exitCode === 0 ? result.stdout.trim().match(/\d+\.\d+\.\d+(?:[-+][\w.-]+)?/)?.[0] : undefined; if (version !== OFFICIAL_CLI_VERSION) { - throw new LibTVExportError('CLI_BOOTSTRAP_FAILED', 'verified LibTV CLI cache returned an unexpected version'); + throw new LibTVExportError('CLI_BOOTSTRAP_FAILED', '经过校验的 LibTV 命令行工具缓存返回了非预期版本'); } return { runner, version }; } -async function ensureAuthenticated(runner, nonInteractive) { +async function ensureAuthenticated(runner, nonInteractive, onProgress) { const probe = await runner.capture(['account', 'info']); - if (probe.exitCode === 0) return; + if (probe.exitCode === 0) { + reportProgress(onProgress, 'LibTV 登录状态有效'); + return { loginPerformed: false }; + } if (nonInteractive) { - throw new LibTVExportError('AUTH_REQUIRED', 'LibTV authentication is required; run `libtv login web --open` first'); + throw new LibTVExportError( + 'AUTH_REQUIRED', + '需要完成 LibTV 授权；请在交互终端中重新运行，或先执行 `libtv login web --open`', + ); } + reportProgress(onProgress, '未检测到有效的 LibTV 登录状态，正在打开浏览器完成 OAuth 授权…'); const login = await runner.interactive(['login', 'web', '--open']); if (login.exitCode === 130 || login.signal) { - throw new LibTVExportError('LOGIN_CANCELLED', 'LibTV browser login was cancelled'); + throw new LibTVExportError('LOGIN_CANCELLED', 'LibTV 浏览器授权已取消'); } if (login.exitCode !== 0) { - throw new LibTVExportError('LOGIN_FAILED', 'LibTV browser login did not complete'); + throw new LibTVExportError('LOGIN_FAILED', 'LibTV 浏览器授权未完成'); } + reportProgress(onProgress, '浏览器授权已完成，正在确认 LibTV 登录状态…'); const verified = await runner.capture(['account', 'info']); if (verified.exitCode !== 0) { - throw new LibTVExportError('LOGIN_FAILED', 'LibTV credentials were not available after browser login'); + throw new LibTVExportError('LOGIN_FAILED', '浏览器授权后仍未获取到可用的 LibTV 登录凭据'); } + reportProgress(onProgress, 'LibTV 授权成功'); + return { loginPerformed: true }; +} + +async function prepareAuthenticatedLibTVCLI(options = {}) { + reportProgress(options.onProgress, '阶段：准备经过校验的 LibTV 命令行工具'); + const located = await locateLibTVCLI({ + binary: options.binary, + env: options.env ?? process.env, + bootstrap: options.bootstrap, + cacheRoot: options.cacheRoot, + }); + reportProgress(options.onProgress, '阶段：检查 LibTV 登录状态'); + const authentication = await ensureAuthenticated( + located.runner, + Boolean(options.nonInteractive), + options.onProgress, + ); + return { ...located, ...authentication }; +} + +async function preflightLibTVAuth(options = {}) { + const { version, loginPerformed } = await prepareAuthenticatedLibTVCLI(options); + return { + schema: AUTH_RESULT_SCHEMA, + provider: 'libtv', + authenticated: true, + cli_version: version, + login_performed: loginPerformed, + }; } function parseCommandJSON(result, commandName) { if (result.exitCode !== 0) { - throw new LibTVExportError('COMMAND_FAILED', `${commandName} failed (exit ${result.exitCode ?? 'spawn'})`); + throw new LibTVExportError('COMMAND_FAILED', `${commandName} 执行失败（退出状态：${result.exitCode ?? '无法启动'}）`); } - if (result.overflow) throw new LibTVExportError('COMMAND_OUTPUT_TOO_LARGE', `${commandName} output exceeded the safety limit`); + if (result.overflow) throw new LibTVExportError('COMMAND_OUTPUT_TOO_LARGE', `${commandName} 的输出超过安全上限`); try { return JSON.parse(result.stdout); } catch { - throw new LibTVExportError('INVALID_CLI_JSON', `${commandName} did not return valid JSON`); + throw new LibTVExportError('INVALID_CLI_JSON', `${commandName} 未返回有效的 JSON`); } } function validateProject(project, projectId) { if (project?.projectUuid !== projectId || !Array.isArray(project?.nodes) || !Array.isArray(project?.edges)) { - throw new LibTVExportError('INVALID_PROJECT', 'LibTV project summary is incomplete or does not match the URL'); + throw new LibTVExportError('INVALID_PROJECT', 'LibTV 项目信息不完整，或与画布链接不匹配'); } const seen = new Set(); for (const [index, node] of project.nodes.entries()) { if (typeof node?.id !== 'string' || !node.id.trim() || seen.has(node.id)) { - throw new LibTVExportError('INVALID_PROJECT', `LibTV project node ${index} has a missing or duplicate ID`); + throw new LibTVExportError('INVALID_PROJECT', `LibTV 项目中的第 ${index + 1} 个节点缺少 ID 或 ID 重复`); } if (!SUPPORTED_NODE_TYPES.has(node.type)) { - throw new LibTVExportError('UNSUPPORTED_NODE', `unsupported LibTV node type: ${node.type ?? '<missing>'}`); + throw new LibTVExportError('UNSUPPORTED_NODE', `暂不支持该 LibTV 节点类型：${node.type ?? '未提供类型'}`); } seen.add(node.id); } @@ -269,10 +308,10 @@ async function regularFilesUnder(root, current = root) { const files = []; for (const entry of await readdir(current, { withFileTypes: true })) { const path = join(current, entry.name); - if (entry.isSymbolicLink()) throw new LibTVExportError('UNSAFE_DOWNLOAD', 'LibTV download produced a symbolic link'); + if (entry.isSymbolicLink()) throw new LibTVExportError('UNSAFE_DOWNLOAD', 'LibTV 下载结果中包含不安全的符号链接'); if (entry.isDirectory()) files.push(...await regularFilesUnder(root, path)); else if (entry.isFile()) files.push(path); - else throw new LibTVExportError('UNSAFE_DOWNLOAD', 'LibTV download produced an unsupported filesystem entry'); + else throw new LibTVExportError('UNSAFE_DOWNLOAD', 'LibTV 下载结果中包含不支持的文件类型'); } return files; } @@ -306,16 +345,16 @@ async function exportMedia(runner, tasks, projectId, stagingPath, onProgress) { const manifest = []; const deduplicated = new Map(); if (tasks.length === 0) { - reportProgress(onProgress, 'media downloads: processed=0/0, remaining=0'); + reportProgress(onProgress, '素材下载进度：已完成 0/0，剩余 0'); } for (const [index, task] of tasks.entries()) { reportProgress( onProgress, - `media download start: current=${index + 1}/${tasks.length}, processed=${index}, ` + - `remaining=${tasks.length - index - 1}`, + `开始下载素材：当前 ${index + 1}/${tasks.length}，已完成 ${index}，` + + `剩余 ${tasks.length - index - 1}`, ); let downloaded; - let lastFailure = 'command failed'; + let lastFailure = '命令执行失败'; for (let attempt = 0; attempt < MEDIA_DOWNLOAD_ATTEMPTS; attempt += 1) { const downloadDirectory = join( downloadsDirectory, @@ -331,12 +370,12 @@ async function exportMedia(runner, tasks, projectId, stagingPath, onProgress) { downloaded = { path: files[0], fileInfo }; break; } - lastFailure = 'produced an invalid media file'; + lastFailure = '生成了无效的素材文件'; } else { - lastFailure = 'did not produce one direct media file'; + lastFailure = '没有生成唯一且可直接使用的素材文件'; } } else { - lastFailure = `failed with exit ${result.exitCode ?? 'spawn'}`; + lastFailure = `退出状态为 ${result.exitCode ?? '无法启动'}`; } if (attempt + 1 < MEDIA_DOWNLOAD_ATTEMPTS) { await new Promise((resolveDelay) => setTimeout(resolveDelay, 200 * (2 ** attempt))); @@ -345,7 +384,7 @@ async function exportMedia(runner, tasks, projectId, stagingPath, onProgress) { if (!downloaded) { throw new LibTVExportError( 'MEDIA_DOWNLOAD_FAILED', - `LibTV ${task.node.type} download failed for node ${task.node.id} after ${MEDIA_DOWNLOAD_ATTEMPTS} attempts (${lastFailure})`, + `LibTV 节点 ${task.node.id} 的 ${task.node.type} 素材连续下载 ${MEDIA_DOWNLOAD_ATTEMPTS} 次仍失败（${lastFailure}）`, ); } const digest = await fileSHA256(downloaded.path); @@ -370,7 +409,7 @@ async function exportMedia(runner, tasks, projectId, stagingPath, onProgress) { }); reportProgress( onProgress, - `media downloads: processed=${index + 1}/${tasks.length}, remaining=${tasks.length - index - 1}`, + `素材下载进度：已完成 ${index + 1}/${tasks.length}，剩余 ${tasks.length - index - 1}`, ); } await rm(downloadsDirectory, { recursive: true, force: true }); @@ -381,27 +420,26 @@ async function exportLibTVURL(options) { const { projectId } = parseLibTVCanvasURL(options.url); const outputPath = resolve(options.outputDir); if (await pathExists(outputPath)) { - throw new LibTVExportError('OUTPUT_EXISTS', `output directory already exists: ${outputPath}`); + throw new LibTVExportError('OUTPUT_EXISTS', `导出目录已存在：${outputPath}`); } - reportProgress(options.onProgress, 'phase: preparing verified LibTV CLI'); - const { runner, version } = await locateLibTVCLI({ + const { runner, version } = await prepareAuthenticatedLibTVCLI({ binary: options.binary, env: options.env ?? process.env, bootstrap: options.bootstrap, cacheRoot: options.cacheRoot, + nonInteractive: options.nonInteractive, + onProgress: options.onProgress, }); - reportProgress(options.onProgress, 'phase: checking LibTV authentication'); - await ensureAuthenticated(runner, Boolean(options.nonInteractive)); - reportProgress(options.onProgress, 'phase: fetching LibTV project summary'); + reportProgress(options.onProgress, '阶段：正在获取 LibTV 项目信息'); const projectResult = await runner.capture(['project', projectId]); if (projectResult.exitCode !== 0) { - throw new LibTVExportError('PROJECT_FORBIDDEN', 'LibTV project is unavailable or permission was denied'); + throw new LibTVExportError('PROJECT_FORBIDDEN', '无法访问该 LibTV 项目，请确认项目存在且当前账号拥有权限'); } - const project = parseCommandJSON(projectResult, 'libtv project'); + const project = parseCommandJSON(projectResult, 'LibTV 项目查询命令'); validateProject(project, projectId); reportProgress( options.onProgress, - `project summary: nodes=${project.nodes.length}, edges=${project.edges.length}`, + `项目信息：节点 ${project.nodes.length} 个，连线 ${project.edges.length} 条`, ); const nodeDetails = []; @@ -411,14 +449,14 @@ async function exportLibTVURL(options) { const detailCommand = node.type === 'group' ? 'group' : 'node'; const detail = parseCommandJSON( await runner.capture([detailCommand, node.id, '-p', projectId]), - `libtv ${detailCommand} ${node.id}`, + `LibTV 节点详情查询命令（${node.id}）`, ); const downloadable = MEDIA_NODE_TYPES.has(node.type) && hasMediaResult(detail); if (downloadable) mediaTasks.push({ node, detail }); else if (node.type === 'image' || node.type === 'video') { emptyMedia.push({ source_node_id: node.id, media_type: node.type, reason: 'source_has_no_media' }); } else if (node.type === 'audio') { - throw new LibTVExportError('EMPTY_AUDIO', `LibTV audio node ${node.id} has no downloadable media`); + throw new LibTVExportError('EMPTY_AUDIO', `LibTV 音频节点 ${node.id} 没有可下载的素材`); } nodeDetails.push({ sourceNodeId: node.id, @@ -427,7 +465,7 @@ async function exportLibTVURL(options) { }); reportProgress( options.onProgress, - `node details: processed=${index + 1}/${project.nodes.length}, remaining=${project.nodes.length - index - 1}`, + `节点详情进度：已完成 ${index + 1}/${project.nodes.length}，剩余 ${project.nodes.length - index - 1}`, ); } @@ -457,7 +495,7 @@ async function exportLibTVURL(options) { const plan = convertSnapshotToCanvasPlan(snapshot, { mediaManifest, title: options.title }); const serialized = JSON.stringify({ snapshot, mediaManifest, plan }); if (/\b(?:https?|data|blob):/i.test(serialized)) { - throw new LibTVExportError('SANITIZATION_FAILED', 'sanitized LibTV bundle still contains an external URL'); + throw new LibTVExportError('SANITIZATION_FAILED', '清理后的 LibTV 导出数据中仍包含外部链接，已停止导出'); } await writePrivateJSON(join(stagingPath, 'snapshot.json'), snapshot); await writePrivateJSON(join(stagingPath, 'media-manifest.json'), mediaManifest); @@ -466,8 +504,8 @@ async function exportLibTVURL(options) { completed = true; reportProgress( options.onProgress, - `export complete: nodes=${plan.nodes.length}, groups=${plan.groups.length}, edges=${plan.edges.length}, ` + - `media=${media.length}, degradations=${plan.degradations.length}`, + `LibTV 导出完成：节点 ${plan.nodes.length} 个，分组 ${plan.groups.length} 个，连线 ${plan.edges.length} 条，` + + `素材 ${media.length} 个，兼容性降级 ${plan.degradations.length} 项`, ); return { schema: EXPORT_RESULT_SCHEMA, @@ -494,6 +532,7 @@ async function exportLibTVURL(options) { } export { + AUTH_RESULT_SCHEMA, EXPORT_RESULT_SCHEMA, MEDIA_MANIFEST_SCHEMA, LibTVExportError, @@ -502,5 +541,6 @@ export { exportLibTVURL, locateLibTVCLI, parseLibTVCanvasURL, + preflightLibTVAuth, sanitizedChildEnvironment, }; diff --git a/adapters/libtv/exporter.test.mjs b/adapters/libtv/exporter.test.mjs index 7a1cd20..44f330f 100644 --- a/adapters/libtv/exporter.test.mjs +++ b/adapters/libtv/exporter.test.mjs @@ -6,11 +6,13 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { + AUTH_RESULT_SCHEMA, EXPORT_RESULT_SCHEMA, MEDIA_MANIFEST_SCHEMA, exportLibTVURL, locateLibTVCLI, parseLibTVCanvasURL, + preflightLibTVAuth, sanitizedChildEnvironment, } from './exporter.mjs'; @@ -165,6 +167,97 @@ async function testBrowserLogin() { } } +async function testAuthPreflightOnlyChecksAccount() { + const context = await fixture('login-required'); + try { + await context.configure({ loginPrompt: true }); + const result = spawnSync(process.execPath, [ + adapterCLI, + 'auth', + '--libtv-cli', fakeCLI, + ], { encoding: 'utf8', env: context.options.env }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout.trim().split('\n').length, 1); + assert.deepEqual(JSON.parse(result.stdout), { + schema: AUTH_RESULT_SCHEMA, + provider: 'libtv', + authenticated: true, + cli_version: '1.1.3', + login_performed: true, + }); + assert.match(result.stderr, /fake browser login prompt/); + assert.deepEqual( + result.stderr.trim().split('\n').filter((line) => line.startsWith('[libtv]')), + [ + '[libtv] 阶段：准备经过校验的 LibTV 命令行工具', + '[libtv] 阶段：检查 LibTV 登录状态', + '[libtv] 未检测到有效的 LibTV 登录状态，正在打开浏览器完成 OAuth 授权…', + '[libtv] 浏览器授权已完成，正在确认 LibTV 登录状态…', + '[libtv] LibTV 授权成功', + ], + ); + const commands = (await readFile(context.logPath, 'utf8')).trim().split('\n').map(JSON.parse); + assert.deepEqual(commands, [ + ['--version'], + ['account', 'info'], + ['login', 'web', '--open'], + ['account', 'info'], + ]); + assert.equal( + commands.some((args) => ['project', 'node', 'group', 'download'].includes(args[0])), + false, + ); + assert.equal(await exists(context.outputDir), false); + } finally { + await rm(context.root, { recursive: true, force: true }); + } +} + +async function testAuthenticatedPreflightSkipsLogin() { + const context = await fixture('authenticated'); + try { + const progress = []; + const result = await preflightLibTVAuth({ + ...context.options, + onProgress: (message) => progress.push(message), + }); + assert.equal(result.authenticated, true); + assert.equal(result.login_performed, false); + assert.deepEqual(progress, [ + '[libtv] 阶段：准备经过校验的 LibTV 命令行工具', + '[libtv] 阶段：检查 LibTV 登录状态', + '[libtv] LibTV 登录状态有效', + ]); + const commands = (await readFile(context.logPath, 'utf8')).trim().split('\n').map(JSON.parse); + assert.deepEqual(commands, [['--version'], ['account', 'info']]); + } finally { + await rm(context.root, { recursive: true, force: true }); + } +} + +async function testNonInteractivePreflightStopsBeforeProject() { + const context = await fixture('non-interactive'); + try { + const result = spawnSync(process.execPath, [ + adapterCLI, + 'auth', + '--libtv-cli', fakeCLI, + '--non-interactive', + ], { encoding: 'utf8', env: context.options.env }); + assert.equal(result.status, 1); + assert.equal(result.stdout, ''); + assert.match(result.stderr, /需要完成 LibTV 授权/); + const commands = (await readFile(context.logPath, 'utf8')).trim().split('\n').map(JSON.parse); + assert.deepEqual(commands, [['--version'], ['account', 'info']]); + assert.equal( + commands.some((args) => ['login', 'project', 'node', 'group', 'download'].includes(args[0])), + false, + ); + } finally { + await rm(context.root, { recursive: true, force: true }); + } +} + async function testTransientMediaRetry() { const context = await fixture('transient-media'); try { @@ -196,22 +289,25 @@ async function testLoginDoesNotPolluteJSONStdout() { assert.deepEqual( result.stderr.trim().split('\n').filter((line) => line.startsWith('[libtv]')), [ - '[libtv] phase: preparing verified LibTV CLI', - '[libtv] phase: checking LibTV authentication', - '[libtv] phase: fetching LibTV project summary', - '[libtv] project summary: nodes=5, edges=1', - '[libtv] node details: processed=1/5, remaining=4', - '[libtv] node details: processed=2/5, remaining=3', - '[libtv] node details: processed=3/5, remaining=2', - '[libtv] node details: processed=4/5, remaining=1', - '[libtv] node details: processed=5/5, remaining=0', - '[libtv] media download start: current=1/3, processed=0, remaining=2', - '[libtv] media downloads: processed=1/3, remaining=2', - '[libtv] media download start: current=2/3, processed=1, remaining=1', - '[libtv] media downloads: processed=2/3, remaining=1', - '[libtv] media download start: current=3/3, processed=2, remaining=0', - '[libtv] media downloads: processed=3/3, remaining=0', - '[libtv] export complete: nodes=4, groups=1, edges=1, media=3, degradations=1', + '[libtv] 阶段：准备经过校验的 LibTV 命令行工具', + '[libtv] 阶段：检查 LibTV 登录状态', + '[libtv] 未检测到有效的 LibTV 登录状态，正在打开浏览器完成 OAuth 授权…', + '[libtv] 浏览器授权已完成，正在确认 LibTV 登录状态…', + '[libtv] LibTV 授权成功', + '[libtv] 阶段：正在获取 LibTV 项目信息', + '[libtv] 项目信息：节点 5 个，连线 1 条', + '[libtv] 节点详情进度：已完成 1/5，剩余 4', + '[libtv] 节点详情进度：已完成 2/5，剩余 3', + '[libtv] 节点详情进度：已完成 3/5，剩余 2', + '[libtv] 节点详情进度：已完成 4/5，剩余 1', + '[libtv] 节点详情进度：已完成 5/5，剩余 0', + '[libtv] 开始下载素材：当前 1/3，已完成 0，剩余 2', + '[libtv] 素材下载进度：已完成 1/3，剩余 2', + '[libtv] 开始下载素材：当前 2/3，已完成 1，剩余 1', + '[libtv] 素材下载进度：已完成 2/3，剩余 1', + '[libtv] 开始下载素材：当前 3/3，已完成 2，剩余 0', + '[libtv] 素材下载进度：已完成 3/3，剩余 0', + '[libtv] LibTV 导出完成：节点 4 个，分组 1 个，连线 1 条，素材 3 个，兼容性降级 1 项', ], ); } finally { @@ -281,11 +377,11 @@ async function testClosedFailure(scenario, expected, extra = {}) { } async function testFailureModes() { - await testClosedFailure('non-interactive', /authentication is required/, { nonInteractive: true }); - await testClosedFailure('login-cancel', /login was cancelled/); - await testClosedFailure('permission-denied', /permission was denied/); - await testClosedFailure('partial-media', /download failed/); - assert.throws(() => parseLibTVCanvasURL('https://evil.example/canvas?projectId=0123456789abcdef0123456789abcdef'), /expected an HTTPS LibTV canvas URL/); + await testClosedFailure('non-interactive', /需要完成 LibTV 授权/, { nonInteractive: true }); + await testClosedFailure('login-cancel', /授权已取消/); + await testClosedFailure('permission-denied', /当前账号拥有权限/); + await testClosedFailure('partial-media', /连续下载 3 次仍失败/); + assert.throws(() => parseLibTVCanvasURL('https://evil.example/canvas?projectId=0123456789abcdef0123456789abcdef'), /请输入 HTTPS 格式/); assert.throws(() => parseLibTVCanvasURL('https://www.liblib.tv/canvas?projectId=bad'), /projectId/); } @@ -293,6 +389,9 @@ await chmod(fakeCLI, 0o755); testChildEnvironmentAllowlist(); await testAuthenticatedExport(); await testBrowserLogin(); +await testAuthPreflightOnlyChecksAccount(); +await testAuthenticatedPreflightSkipsLogin(); +await testNonInteractivePreflightStopsBeforeProject(); await testTransientMediaRetry(); await testLoginDoesNotPolluteJSONStdout(); await testLocateUsesVerifiedBootstrap(); diff --git a/cmd/canvas/import_export.go b/cmd/canvas/import_export.go index 5d87c06..195b32a 100644 --- a/cmd/canvas/import_export.go +++ b/cmd/canvas/import_export.go @@ -12,6 +12,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strings" "github.com/Pippit-dev/pippit-cli/internal/canvasplan" @@ -19,9 +20,20 @@ import ( const ( maxLibTVExporterOutputBytes = 4 << 20 + libTVAuthResultSchema = "pippit-libtv-auth-result/0.1" libTVExportResultSchema = "pippit-libtv-export-result/0.1" ) +var libTVCLIVersionPattern = regexp.MustCompile(`^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$`) + +type libTVAuthResult struct { + Schema string `json:"schema"` + Provider string `json:"provider"` + Authenticated bool `json:"authenticated"` + CLIVersion string `json:"cli_version"` + LoginPerformed bool `json:"login_performed"` +} + type libTVExportResult struct { BundleDir string `json:"bundle_dir"` SnapshotPath string `json:"snapshot_path"` @@ -46,47 +58,111 @@ type libTVExportMedia struct { type nodeLibTVExporter struct{} +func (nodeLibTVExporter) Authenticate( + ctx context.Context, + interactive bool, + stderr io.Writer, +) error { + args := []string{"auth"} + var stdin io.Reader + if interactive { + stdin = os.Stdin + } else { + args = append(args, "--non-interactive") + } + stdout, err := runNodeLibTVAdapter(ctx, args, stdin, stderr) + if err != nil { + return fmt.Errorf("检查 LibTV 授权失败：%w", err) + } + if _, err := decodeLibTVAuthResult(stdout); err != nil { + return fmt.Errorf("解析 LibTV 授权结果失败：%w", err) + } + return nil +} + func (nodeLibTVExporter) Export( ctx context.Context, sourceURL string, outputDir string, stderr io.Writer, ) (*libTVExportResult, error) { + stdout, err := runNodeLibTVAdapter( + ctx, + []string{"export", "--url", sourceURL, "--output-dir", outputDir, "--non-interactive"}, + nil, + stderr, + ) + if err != nil { + return nil, fmt.Errorf("运行 LibTV 导出器失败：%w", err) + } + var result libTVExportResult + decoder := json.NewDecoder(bytes.NewReader(stdout)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&result); err != nil { + return nil, fmt.Errorf("解析 LibTV 导出结果失败：%w", err) + } + if err := ensureImportJSONEOF(decoder); err != nil { + return nil, fmt.Errorf("解析 LibTV 导出结果失败：%w", err) + } + return &result, nil +} + +func runNodeLibTVAdapter( + ctx context.Context, + args []string, + stdin io.Reader, + stderr io.Writer, +) ([]byte, error) { root, err := findCLIPackageRoot() if err != nil { return nil, err } node, err := exec.LookPath("node") if err != nil { - return nil, fmt.Errorf("find Node.js for LibTV exporter: %w", err) + return nil, fmt.Errorf("未找到 LibTV 适配器所需的 Node.js：%w", err) } adapterPath := filepath.Join(root, "adapters", "libtv", "cli.mjs") - command := exec.CommandContext(ctx, node, adapterPath, - "export", "--url", sourceURL, "--output-dir", outputDir, - ) + commandArgs := append([]string{adapterPath}, args...) + command := exec.CommandContext(ctx, node, commandArgs...) command.Env = sanitizedExporterEnv(os.Environ()) - command.Stdin = os.Stdin + command.Stdin = stdin command.Stderr = stderr var stdout boundedBuffer stdout.maximum = maxLibTVExporterOutputBytes command.Stdout = &stdout if err := command.Run(); err != nil { if stdout.exceeded { - return nil, fmt.Errorf("LibTV exporter output exceeds %d bytes", maxLibTVExporterOutputBytes) + return nil, fmt.Errorf("LibTV 适配器输出超过 %d 字节安全上限", maxLibTVExporterOutputBytes) } - return nil, fmt.Errorf("LibTV exporter failed: %w", err) + return nil, fmt.Errorf("LibTV 适配器执行失败：%w", err) } if stdout.exceeded { - return nil, fmt.Errorf("LibTV exporter output exceeds %d bytes", maxLibTVExporterOutputBytes) + return nil, fmt.Errorf("LibTV 适配器输出超过 %d 字节安全上限", maxLibTVExporterOutputBytes) } - var result libTVExportResult - decoder := json.NewDecoder(bytes.NewReader(stdout.Bytes())) + return append([]byte(nil), stdout.Bytes()...), nil +} + +func decodeLibTVAuthResult(data []byte) (*libTVAuthResult, error) { + var result libTVAuthResult + decoder := json.NewDecoder(bytes.NewReader(data)) decoder.DisallowUnknownFields() if err := decoder.Decode(&result); err != nil { - return nil, fmt.Errorf("decode LibTV exporter result: %w", err) + return nil, fmt.Errorf("授权结果不是有效的 JSON：%w", err) } if err := ensureImportJSONEOF(decoder); err != nil { - return nil, fmt.Errorf("decode LibTV exporter result: %w", err) + return nil, err + } + if result.Schema != libTVAuthResultSchema { + return nil, fmt.Errorf("授权结果 schema 无效：%q", result.Schema) + } + if result.Provider != "libtv" { + return nil, fmt.Errorf("授权结果 provider 无效：%q", result.Provider) + } + if !result.Authenticated { + return nil, fmt.Errorf("LibTV 授权尚未完成") + } + if !libTVCLIVersionPattern.MatchString(result.CLIVersion) { + return nil, fmt.Errorf("授权结果中的 LibTV CLI 版本无效：%q", result.CLIVersion) } return &result, nil } @@ -96,9 +172,9 @@ func ensureImportJSONEOF(decoder *json.Decoder) error { if err := decoder.Decode(&trailing); err == io.EOF { return nil } else if err != nil { - return fmt.Errorf("decode trailing JSON: %w", err) + return fmt.Errorf("解析末尾 JSON 失败：%w", err) } - return fmt.Errorf("JSON input must contain exactly one value") + return fmt.Errorf("JSON 输入必须且只能包含一个值") } type boundedBuffer struct { diff --git a/cmd/canvas/import_export_auth_test.go b/cmd/canvas/import_export_auth_test.go new file mode 100644 index 0000000..4bf36b9 --- /dev/null +++ b/cmd/canvas/import_export_auth_test.go @@ -0,0 +1,169 @@ +package canvas + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestDecodeLibTVAuthResultStrictlyValidatesContract(t *testing.T) { + valid := `{"schema":"pippit-libtv-auth-result/0.1","provider":"libtv","authenticated":true,"cli_version":"1.1.3","login_performed":false}` + tests := []struct { + name string + payload string + want string + }{ + {name: "invalid json", payload: `{`, want: "不是有效的 JSON"}, + {name: "trailing json", payload: valid + `{}`, want: "只能包含一个值"}, + {name: "unknown field", payload: strings.Replace(valid, `}`, `,"extra":true}`, 1), want: "unknown field"}, + {name: "wrong schema", payload: strings.Replace(valid, libTVAuthResultSchema, "other/0.1", 1), want: "schema 无效"}, + {name: "wrong provider", payload: strings.Replace(valid, `"provider":"libtv"`, `"provider":"other"`, 1), want: "provider 无效"}, + {name: "not authenticated", payload: strings.Replace(valid, `"authenticated":true`, `"authenticated":false`, 1), want: "授权尚未完成"}, + {name: "missing version", payload: strings.Replace(valid, `"cli_version":"1.1.3"`, `"cli_version":""`, 1), want: "CLI 版本无效"}, + {name: "malformed version", payload: strings.Replace(valid, `"cli_version":"1.1.3"`, `"cli_version":"latest"`, 1), want: "CLI 版本无效"}, + } + + result, err := decodeLibTVAuthResult([]byte(valid)) + if err != nil { + t.Fatalf("decodeLibTVAuthResult(valid) error = %v", err) + } + if result.Provider != "libtv" || !result.Authenticated || result.CLIVersion != "1.1.3" || result.LoginPerformed { + t.Fatalf("decodeLibTVAuthResult(valid) = %#v", result) + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := decodeLibTVAuthResult([]byte(test.payload)) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("decodeLibTVAuthResult() error = %v, want %q", err, test.want) + } + }) + } +} + +func TestNodeLibTVExporterAuthenticateUsesDedicatedAuthCommand(t *testing.T) { + root := t.TempDir() + adapterDir := filepath.Join(root, "adapters", "libtv") + if err := os.MkdirAll(adapterDir, 0o700); err != nil { + t.Fatal(err) + } + logPath := filepath.Join(root, "args.json") + result := map[string]any{ + "schema": libTVAuthResultSchema, + "provider": "libtv", + "authenticated": true, + "cli_version": "1.1.3", + "login_performed": false, + } + resultJSON, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + logLiteral, _ := json.Marshal(logPath) + resultLiteral, _ := json.Marshal(string(resultJSON) + "\n") + script := fmt.Sprintf( + "import { writeFileSync } from 'node:fs';\n"+ + "writeFileSync(%s, JSON.stringify(process.argv.slice(2)));\n"+ + "process.stderr.write('LibTV 授权测试提示\\n');\n"+ + "process.stdout.write(%s);\n", + logLiteral, + resultLiteral, + ) + if err := os.WriteFile(filepath.Join(adapterDir, "cli.mjs"), []byte(script), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("PIPPIT_CLI_PACKAGE_ROOT", root) + + exporter := nodeLibTVExporter{} + var stderr bytes.Buffer + if err := exporter.Authenticate(context.Background(), false, &stderr); err != nil { + t.Fatalf("Authenticate(non-interactive) error = %v; stderr = %s", err, stderr.String()) + } + assertLibTVAdapterArgs(t, logPath, []string{"auth", "--non-interactive"}) + if !strings.Contains(stderr.String(), "LibTV 授权测试提示") { + t.Fatalf("Authenticate() stderr = %q, want adapter progress", stderr.String()) + } + + stderr.Reset() + if err := exporter.Authenticate(context.Background(), true, &stderr); err != nil { + t.Fatalf("Authenticate(interactive) error = %v; stderr = %s", err, stderr.String()) + } + assertLibTVAdapterArgs(t, logPath, []string{"auth"}) +} + +func TestNodeLibTVExporterAuthenticateRejectsInvalidAdapterResult(t *testing.T) { + root := t.TempDir() + adapterDir := filepath.Join(root, "adapters", "libtv") + if err := os.MkdirAll(adapterDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(adapterDir, "cli.mjs"), + []byte("process.stdout.write('{\"schema\":\"wrong\"}\\n');\n"), + 0o600, + ); err != nil { + t.Fatal(err) + } + t.Setenv("PIPPIT_CLI_PACKAGE_ROOT", root) + + err := (nodeLibTVExporter{}).Authenticate(context.Background(), true, &bytes.Buffer{}) + if err == nil || !strings.Contains(err.Error(), "解析 LibTV 授权结果失败") { + t.Fatalf("Authenticate() error = %v, want strict result rejection", err) + } +} + +func TestNodeLibTVExporterExportKeepsDefensiveAuthCheckNonInteractive(t *testing.T) { + root := t.TempDir() + adapterDir := filepath.Join(root, "adapters", "libtv") + if err := os.MkdirAll(adapterDir, 0o700); err != nil { + t.Fatal(err) + } + logPath := filepath.Join(root, "args.json") + resultJSON, err := json.Marshal(libTVExportResult{}) + if err != nil { + t.Fatal(err) + } + logLiteral, _ := json.Marshal(logPath) + resultLiteral, _ := json.Marshal(string(resultJSON) + "\n") + script := fmt.Sprintf( + "import { writeFileSync } from 'node:fs';\n"+ + "writeFileSync(%s, JSON.stringify(process.argv.slice(2)));\n"+ + "process.stdout.write(%s);\n", + logLiteral, + resultLiteral, + ) + if err := os.WriteFile(filepath.Join(adapterDir, "cli.mjs"), []byte(script), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("PIPPIT_CLI_PACKAGE_ROOT", root) + + if _, err := (nodeLibTVExporter{}).Export( + context.Background(), testLibTVURL, filepath.Join(root, "bundle"), &bytes.Buffer{}, + ); err != nil { + t.Fatalf("Export() error = %v", err) + } + assertLibTVAdapterArgs(t, logPath, []string{ + "export", "--url", testLibTVURL, "--output-dir", filepath.Join(root, "bundle"), "--non-interactive", + }) +} + +func assertLibTVAdapterArgs(t *testing.T, path string, expected []string) { + t.Helper() + var actual []string + payload, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(payload, &actual); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("adapter args = %v, want %v", actual, expected) + } +} From 01c472daa4f3ef9ebdd861a045f3637b7d57742b Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 22:47:54 +0800 Subject: [PATCH 32/48] feat(canvas): add secure import credential prompts Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_prompt.go | 60 +++++++++++++++++++--------- cmd/canvas/import_prompt_tui.go | 36 +++++++++++++---- cmd/canvas/import_prompt_tui_test.go | 44 +++++++++++++++----- go.mod | 2 +- 4 files changed, 106 insertions(+), 36 deletions(-) diff --git a/cmd/canvas/import_prompt.go b/cmd/canvas/import_prompt.go index 8aaeff7..48e7d2a 100644 --- a/cmd/canvas/import_prompt.go +++ b/cmd/canvas/import_prompt.go @@ -7,11 +7,14 @@ import ( "io" "os" "strings" + + charmterm "github.com/charmbracelet/x/term" ) const importFlagsHint = `--from libtv --url "https://www.liblib.tv/canvas?projectId=<project-id>"` type importPromptSession struct { + input io.Reader reader *bufio.Reader stderr io.Writer eof bool @@ -45,6 +48,7 @@ func newImportPromptSessionWithTUI( enableTUI bool, ) *importPromptSession { session := &importPromptSession{ + input: input, reader: bufio.NewReader(input), stderr: stderr, } @@ -71,19 +75,22 @@ func prepareCanvasImportOptions( ) (importOptions, *importPromptSession, error) { needsWizard := strings.TrimSpace(opts.Provider) == "" || strings.TrimSpace(opts.SourceURL) == "" if !needsWizard { + if isInteractive != nil && isInteractive(input) { + return opts, newImportPromptSession(ctx, input, stderr), nil + } return opts, nil, nil } if isInteractive == nil || !isInteractive(input) { return opts, nil, fmt.Errorf( - "canvas import is missing --from or --url and stdin is not interactive; pass %s", + "canvas import 缺少 --from 或 --url，且当前输入不是交互式终端；请传入 %s", importFlagsHint, ) } prompts := newImportPromptSession(ctx, input, stderr) if strings.TrimSpace(opts.Provider) == "" { _, err := prompts.askChoice( - "Source provider:", - []importPromptChoice{{label: "LibTV (default)", aliases: []string{"libtv"}}}, + "导入来源：", + []importPromptChoice{{label: "LibTV（默认）", aliases: []string{"libtv"}}}, 1, ) if err != nil { @@ -93,7 +100,7 @@ func prepareCanvasImportOptions( } if strings.TrimSpace(opts.SourceURL) == "" { for { - value, eof, err := prompts.readLine("LibTV canvas URL: ") + value, eof, err := prompts.readLine("LibTV 画布链接：") if err != nil { return opts, nil, err } @@ -103,19 +110,19 @@ func prepareCanvasImportOptions( } if eof { return opts, nil, fmt.Errorf( - "interactive input ended before a LibTV URL was provided; rerun with %s", + "尚未提供 LibTV 画布链接，交互输入就已结束；请重新运行并传入 %s", importFlagsHint, ) } - fmt.Fprintln(stderr, "A LibTV canvas URL is required.") + fmt.Fprintln(stderr, "请输入 LibTV 画布链接。") } } if !opts.JournalExplicit { choice, err := prompts.askChoice( - "Resume journal:", + "断点续跑记录：", []importPromptChoice{ - {label: "Automatic (recommended, default)"}, - {label: "Custom path"}, + {label: "自动生成（推荐，默认）"}, + {label: "自定义路径"}, }, 1, ) @@ -124,7 +131,7 @@ func prepareCanvasImportOptions( } if choice == 2 { for { - value, eof, readErr := prompts.readLine("Custom journal path: ") + value, eof, readErr := prompts.readLine("自定义断点记录路径：") if readErr != nil { return opts, nil, readErr } @@ -135,19 +142,19 @@ func prepareCanvasImportOptions( } if eof { return opts, nil, fmt.Errorf( - "interactive input ended before a custom journal path was provided; choose 1 for Automatic or pass --journal <path>", + "尚未提供自定义断点记录路径，交互输入就已结束；请选择 1 自动生成，或传入 --journal <路径>", ) } - fmt.Fprintln(stderr, "A custom journal path is required after selecting option 2.") + fmt.Fprintln(stderr, "选择自定义路径后，请输入断点记录路径。") } } } if !opts.OpenExplicit { choice, err := prompts.askChoice( - "After import:", + "导入完成后：", []importPromptChoice{ - {label: "Open Canvas (default)", aliases: []string{"y", "yes"}}, - {label: "Do not open", aliases: []string{"n", "no"}}, + {label: "打开画布（默认）", aliases: []string{"y", "yes"}}, + {label: "暂不打开", aliases: []string{"n", "no"}}, }, 1, ) @@ -159,6 +166,23 @@ func prepareCanvasImportOptions( return opts, prompts, nil } +func (prompts *importPromptSession) readSecret(label string) (string, bool, error) { + if prompts.tui != nil { + value, err := prompts.tui.readSecret(label) + return value, false, err + } + if file, ok := prompts.input.(*os.File); ok && importFileIsTerminal(file) { + fmt.Fprint(prompts.stderr, label) + value, err := charmterm.ReadPassword(file.Fd()) + fmt.Fprintln(prompts.stderr) + if err != nil { + return "", false, fmt.Errorf("安全读取 Access Key 失败：%w", err) + } + return strings.TrimSpace(string(value)), false, nil + } + return prompts.readLine(label) +} + func (prompts *importPromptSession) askChoice( title string, choices []importPromptChoice, @@ -172,7 +196,7 @@ func (prompts *importPromptSession) askChoice( for index, choice := range choices { fmt.Fprintf(prompts.stderr, " %d) %s\n", index+1, choice.label) } - value, eof, err := prompts.readLine(fmt.Sprintf("Select [%d]: ", defaultChoice)) + value, eof, err := prompts.readLine(fmt.Sprintf("请选择 [%d]：", defaultChoice)) if err != nil { return 0, err } @@ -188,7 +212,7 @@ func (prompts *importPromptSession) askChoice( if eof { return defaultChoice, nil } - fmt.Fprintf(prompts.stderr, "Please select a number from 1 to %d.\n", len(choices)) + fmt.Fprintf(prompts.stderr, "请输入 1 到 %d 之间的数字。\n", len(choices)) } } @@ -212,7 +236,7 @@ func (prompts *importPromptSession) readLine(label string) (string, bool, error) } line, err := prompts.reader.ReadString('\n') if err != nil && err != io.EOF { - return "", false, fmt.Errorf("read canvas import prompt: %w", err) + return "", false, fmt.Errorf("读取画布导入提示失败：%w", err) } if err == io.EOF { prompts.eof = true diff --git a/cmd/canvas/import_prompt_tui.go b/cmd/canvas/import_prompt_tui.go index d1fc0e0..92f2e15 100644 --- a/cmd/canvas/import_prompt_tui.go +++ b/cmd/canvas/import_prompt_tui.go @@ -12,7 +12,7 @@ import ( const importPromptWidth = 72 -var errCanvasImportSetupCanceled = errors.New("Canvas import setup canceled") +var errCanvasImportSetupCanceled = errors.New("已取消画布导入设置") type importPromptTUI struct { ctx context.Context @@ -26,7 +26,7 @@ func (prompt *importPromptTUI) askChoice( defaultChoice int, ) (int, error) { if defaultChoice < 1 || defaultChoice > len(choices) { - return 0, fmt.Errorf("invalid default Canvas import choice %d", defaultChoice) + return 0, fmt.Errorf("画布导入的默认选项 %d 无效", defaultChoice) } selected := defaultChoice options := make([]huh.Option[int], 0, len(choices)) @@ -35,7 +35,7 @@ func (prompt *importPromptTUI) askChoice( } field := huh.NewSelect[int](). Title(strings.TrimSpace(title)). - Description("Use ↑/↓ to move, Enter to select"). + Description("使用 ↑/↓ 切换，按 Enter 确认"). Options(options...). Value(&selected) if err := prompt.run(field); err != nil { @@ -49,21 +49,21 @@ func (prompt *importPromptTUI) readLine(label string) (string, error) { title := strings.TrimSpace(strings.TrimSuffix(label, ": ")) field := huh.NewInput(). Title(title). - Description("Paste a value, then press Enter"). + Description("粘贴内容后按 Enter 确认"). Prompt("› "). Value(&value) switch title { - case "LibTV canvas URL": + case "LibTV 画布链接": field.Validate(func(value string) error { if _, err := normalizeLibTVURL(value); err != nil { return err } return nil }) - case "Custom journal path": + case "自定义断点记录路径": field.Validate(func(value string) error { if strings.TrimSpace(value) == "" { - return fmt.Errorf("a custom journal path is required") + return fmt.Errorf("请输入自定义断点记录路径") } return nil }) @@ -74,6 +74,26 @@ func (prompt *importPromptTUI) readLine(label string) (string, error) { return strings.TrimSpace(value), nil } +func (prompt *importPromptTUI) readSecret(label string) (string, error) { + value := "" + field := huh.NewInput(). + Title(strings.TrimSpace(strings.TrimRight(label, ":： "))). + Description("粘贴后按 Enter 确认；内容仅用于当前进程，不会保存"). + Prompt("› "). + EchoMode(huh.EchoModePassword). + Validate(func(value string) error { + if strings.TrimSpace(value) == "" { + return fmt.Errorf("Access Key 不能为空") + } + return nil + }). + Value(&value) + if err := prompt.run(field); err != nil { + return "", err + } + return strings.TrimSpace(value), nil +} + func (prompt *importPromptTUI) run(field huh.Field) error { form := huh.NewForm(huh.NewGroup(field)). WithTheme(huh.ThemeCharm()). @@ -91,7 +111,7 @@ func (prompt *importPromptTUI) run(field huh.Field) error { if errors.Is(err, huh.ErrUserAborted) { return errCanvasImportSetupCanceled } - return fmt.Errorf("run Canvas import terminal prompt: %w", err) + return fmt.Errorf("运行画布导入终端交互失败：%w", err) } return nil } diff --git a/cmd/canvas/import_prompt_tui_test.go b/cmd/canvas/import_prompt_tui_test.go index d77e18c..b900305 100644 --- a/cmd/canvas/import_prompt_tui_test.go +++ b/cmd/canvas/import_prompt_tui_test.go @@ -33,10 +33,10 @@ func TestImportPromptTUISelectUsesArrowKeys(t *testing.T) { true, ) selected, err := session.askChoice( - "Resume journal:", + "断点续跑记录：", []importPromptChoice{ - {label: "Automatic (recommended)"}, - {label: "Custom path"}, + {label: "自动生成（推荐）"}, + {label: "自定义路径"}, }, 1, ) @@ -46,7 +46,7 @@ func TestImportPromptTUISelectUsesArrowKeys(t *testing.T) { if selected != 2 { t.Fatalf("askChoice() = %d, want arrow-down selection 2", selected) } - for _, expected := range []string{"Resume journal", "Automatic", "Custom path"} { + for _, expected := range []string{"断点续跑记录", "自动生成", "自定义路径", "使用 ↑/↓ 切换，按 Enter 确认"} { if !strings.Contains(output.String(), expected) { t.Fatalf("TUI output missing %q: %q", expected, output.String()) } @@ -64,7 +64,7 @@ func TestImportPromptTUIReadsPastedURL(t *testing.T) { &output, true, ) - value, eof, err := session.readLine("LibTV canvas URL: ") + value, eof, err := session.readLine("LibTV 画布链接：") if err != nil { t.Fatalf("readLine() error = %v; output = %q", err, output.String()) } @@ -74,8 +74,34 @@ func TestImportPromptTUIReadsPastedURL(t *testing.T) { if value != testLibTVURL { t.Fatalf("readLine() = %q, want pasted URL", value) } - if !strings.Contains(output.String(), "LibTV canvas URL") { - t.Fatalf("TUI output = %q, want input title", output.String()) + for _, expected := range []string{"LibTV 画布链接", "粘贴内容后按 Enter 确认"} { + if !strings.Contains(output.String(), expected) { + t.Fatalf("TUI output missing %q: %q", expected, output.String()) + } + } +} + +func TestImportPromptTUIKeepsAccessKeyMasked(t *testing.T) { + t.Setenv("TERM", "xterm-256color") + const accessKey = "tui-secret-access-key" + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + var output bytes.Buffer + session := newImportPromptSessionWithTUI( + ctx, + &delayedImportPromptReader{Reader: bytes.NewBufferString(accessKey + "\r")}, + &output, + true, + ) + value, eof, err := session.readSecret("粘贴小云雀 Access Key：") + if err != nil { + t.Fatalf("readSecret() error = %v; output = %q", err, output.String()) + } + if eof || value != accessKey { + t.Fatalf("readSecret() = %q/%v, want masked value", value, eof) + } + if strings.Contains(output.String(), accessKey) { + t.Fatalf("TUI output leaked Access Key: %q", output.String()) } } @@ -86,8 +112,8 @@ func TestImportPromptTUIHonorsContextCancellation(t *testing.T) { var output bytes.Buffer session := newImportPromptSessionWithTUI(ctx, bytes.NewBuffer(nil), &output, true) _, err := session.askChoice( - "After import:", - []importPromptChoice{{label: "Open Canvas"}, {label: "Do not open"}}, + "导入完成后：", + []importPromptChoice{{label: "打开画布"}, {label: "暂不打开"}}, 1, ) if err == nil || !errors.Is(err, errCanvasImportSetupCanceled) { diff --git a/go.mod b/go.mod index 0e0474b..26c14fc 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.0 require ( github.com/bytedance/sonic v1.15.1 github.com/charmbracelet/huh v1.0.0 + github.com/charmbracelet/x/term v0.2.1 github.com/spf13/cobra v1.8.1 golang.org/x/sys v0.33.0 ) @@ -22,7 +23,6 @@ require ( github.com/charmbracelet/x/ansi v0.9.3 // indirect github.com/charmbracelet/x/cellbuf v0.0.13 // indirect github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect - github.com/charmbracelet/x/term v0.2.1 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect From 0a21fca6be2c56d15db8cbe5d53610e5dbcfc1ee Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 22:48:17 +0800 Subject: [PATCH 33/48] feat(canvas): validate Pippit import credentials Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_auth.go | 253 ++++++++++++++++++++++++++++ cmd/canvas/import_auth_test.go | 294 +++++++++++++++++++++++++++++++++ 2 files changed, 547 insertions(+) create mode 100644 cmd/canvas/import_auth.go create mode 100644 cmd/canvas/import_auth_test.go diff --git a/cmd/canvas/import_auth.go b/cmd/canvas/import_auth.go new file mode 100644 index 0000000..efb1f57 --- /dev/null +++ b/cmd/canvas/import_auth.go @@ -0,0 +1,253 @@ +package canvas + +import ( + "context" + "errors" + "fmt" + "strings" + + canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" + "github.com/Pippit-dev/pippit-cli/internal/common" +) + +const canvasImportAuthProbeAssetID = "9223372036854775807" + +var errCanvasImportAuthCanceled = errors.New("已取消小云雀授权") +var errCanvasImportReauthenticationRequired = errors.New("需要重新校验小云雀授权") + +type importAuthAPI interface { + AccessKey() string + SetAccessKey(string) error + Probe(context.Context) error +} + +type runnerImportAuthAPI struct { + runner *common.Runner +} + +func (api runnerImportAuthAPI) AccessKey() string { + if api.runner == nil || api.runner.Config == nil { + return "" + } + return strings.TrimSpace(api.runner.Config.AccessKey) +} + +func (api runnerImportAuthAPI) SetAccessKey(accessKey string) error { + if api.runner == nil || api.runner.Config == nil { + return fmt.Errorf("小云雀 CLI 运行时配置不完整") + } + api.runner.Config.AccessKey = strings.TrimSpace(accessKey) + return nil +} + +func (api runnerImportAuthAPI) Probe(ctx context.Context) error { + if api.runner == nil || api.runner.Config == nil || api.runner.Client == nil { + return fmt.Errorf("小云雀 CLI 运行时配置不完整") + } + var response struct { + Ret string `json:"ret"` + } + err := api.runner.Client.SendRequest(ctx, canvascore.QueryPath, map[string]any{ + "pippit_asset_ids": []string{canvasImportAuthProbeAssetID}, + "Base": map[string]any{}, + }, &response) + if err != nil { + return err + } + if strings.TrimSpace(response.Ret) != "0" { + return fmt.Errorf("小云雀授权校验返回业务错误（ret=%s）", strings.TrimSpace(response.Ret)) + } + return nil +} + +type importAuthPromptAction uint8 + +const ( + importAuthPromptRetry importAuthPromptAction = iota + 1 + importAuthPromptReplace + importAuthPromptCancel +) + +type importAuthPromptRequest struct { + HasAccessKey bool + Failure string +} + +type importAuthPromptResponse struct { + Action importAuthPromptAction + AccessKey string +} + +type importAuthPrompt func(context.Context, importAuthPromptRequest) (importAuthPromptResponse, error) + +// ensureCanvasImportPippitAuth verifies Pippit authorization before the source +// export starts. Interactive callers may retry a transient failure, replace an +// invalid key in memory, or cancel. The key is never persisted by this flow. +func ensureCanvasImportPippitAuth( + ctx context.Context, + auth importAuthAPI, + interactive bool, + prompt importAuthPrompt, +) error { + if err := ctx.Err(); err != nil { + return err + } + if auth == nil { + return fmt.Errorf("小云雀授权检查未配置") + } + + accessKey := strings.TrimSpace(auth.AccessKey()) + failure := "" + for { + if accessKey == "" { + if !interactive { + return fmt.Errorf("未找到小云雀 Access Key；请先设置 XYQ_ACCESS_KEY，或在交互模式中安全粘贴 Access Key") + } + if prompt == nil { + return fmt.Errorf("未找到小云雀 Access Key，且交互授权引导未配置") + } + response, err := prompt(ctx, importAuthPromptRequest{ + HasAccessKey: false, + Failure: failure, + }) + if err != nil { + return canvasImportAuthPromptError(err) + } + switch response.Action { + case importAuthPromptCancel: + return errCanvasImportAuthCanceled + case importAuthPromptReplace: + accessKey = strings.TrimSpace(response.AccessKey) + if accessKey == "" { + failure = "Access Key 不能为空，请重新粘贴" + continue + } + if err := auth.SetAccessKey(accessKey); err != nil { + return fmt.Errorf("更新小云雀内存授权信息失败：%s", redactCanvasImportAuthFailure(err, accessKey)) + } + case importAuthPromptRetry: + failure = "当前没有可重试的 Access Key，请先粘贴" + continue + default: + return fmt.Errorf("小云雀授权引导返回了未知操作") + } + } + + if err := ctx.Err(); err != nil { + return err + } + if err := auth.Probe(ctx); err == nil { + return nil + } else { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + failure = describeCanvasImportAuthProbeFailure(err, accessKey) + } + if !interactive { + return fmt.Errorf("小云雀 Access Key 校验失败：%s", failure) + } + if prompt == nil { + return fmt.Errorf("小云雀 Access Key 校验失败，且交互授权引导未配置：%s", failure) + } + + response, err := prompt(ctx, importAuthPromptRequest{ + HasAccessKey: true, + Failure: failure, + }) + if err != nil { + return canvasImportAuthPromptError(err) + } + switch response.Action { + case importAuthPromptRetry: + continue + case importAuthPromptReplace: + replacement := strings.TrimSpace(response.AccessKey) + if replacement == "" { + accessKey = "" + failure = "Access Key 不能为空，请重新粘贴" + continue + } + if err := auth.SetAccessKey(replacement); err != nil { + return fmt.Errorf("更新小云雀内存授权信息失败：%s", redactCanvasImportAuthFailure(err, replacement)) + } + accessKey = replacement + case importAuthPromptCancel: + return errCanvasImportAuthCanceled + default: + return fmt.Errorf("小云雀授权引导返回了未知操作") + } + } +} + +func canvasImportAuthPromptError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + // Prompt implementations may have already read a candidate key when they + // fail. Do not include their raw error text in user-facing output. + return errors.New("读取小云雀授权选择失败") +} + +func redactCanvasImportAuthFailure(err error, accessKey string) string { + if err == nil { + return "未知错误" + } + message := strings.TrimSpace(err.Error()) + if message == "" { + message = "未知错误" + } + if accessKey = strings.TrimSpace(accessKey); accessKey != "" { + message = strings.ReplaceAll(message, accessKey, "[已隐藏]") + } + return message +} + +func redactCanvasImportFinalError(err error, auth importAuthAPI) error { + if err == nil || auth == nil { + return err + } + original := err.Error() + redacted := redactCanvasImportAuthFailure(err, auth.AccessKey()) + if redacted == original { + return err + } + return errors.New(redacted) +} + +func describeCanvasImportAuthProbeFailure(err error, accessKey string) string { + message := strings.ToLower(redactCanvasImportAuthFailure(err, accessKey)) + switch { + case strings.Contains(message, "http 401"), strings.Contains(message, "unauthorized"): + return "小云雀拒绝了当前 Access Key（HTTP 401），它可能无效、已过期或不属于当前环境" + case strings.Contains(message, "http 403"), strings.Contains(message, "forbidden"): + return "当前 Access Key 无权访问该小云雀环境（HTTP 403）" + case strings.Contains(message, "timeout"), strings.Contains(message, "deadline exceeded"), strings.Contains(message, "超时"): + return "小云雀授权校验请求超时，请检查网络后重试" + default: + return "小云雀授权校验未通过，请检查网络、PPE 环境和 Access Key 后重试" + } +} + +func isCanvasImportPippitAuthFailure(err error) bool { + if err == nil { + return false + } + if errors.Is(err, errCanvasImportReauthenticationRequired) { + return true + } + message := strings.ToLower(err.Error()) + for _, marker := range []string{ + "http 401", "http 403", "unauthorized", "forbidden", + `ret="1015"`, "ret=1015", "xyq_access_key 缺失", "缺少 xyq_access_key", + "access key 校验失败", "access key 无权", + } { + if strings.Contains(message, marker) { + return true + } + } + return false +} diff --git a/cmd/canvas/import_auth_test.go b/cmd/canvas/import_auth_test.go new file mode 100644 index 0000000..7d35251 --- /dev/null +++ b/cmd/canvas/import_auth_test.go @@ -0,0 +1,294 @@ +package canvas + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Pippit-dev/pippit-cli/internal/common" + "github.com/Pippit-dev/pippit-cli/internal/config" +) + +type fakeImportAuthAPI struct { + accessKey string + setValues []string + probeErrors []error + probes int + setErr error +} + +func (api *fakeImportAuthAPI) AccessKey() string { + return api.accessKey +} + +func (api *fakeImportAuthAPI) SetAccessKey(accessKey string) error { + api.setValues = append(api.setValues, accessKey) + if api.setErr != nil { + return api.setErr + } + api.accessKey = accessKey + return nil +} + +func (api *fakeImportAuthAPI) Probe(context.Context) error { + api.probes++ + if len(api.probeErrors) == 0 { + return nil + } + err := api.probeErrors[0] + api.probeErrors = api.probeErrors[1:] + return err +} + +func TestEnsureCanvasImportPippitAuthAcceptsExistingKey(t *testing.T) { + auth := &fakeImportAuthAPI{accessKey: "configured-key"} + prompted := false + + err := ensureCanvasImportPippitAuth( + context.Background(), + auth, + true, + func(context.Context, importAuthPromptRequest) (importAuthPromptResponse, error) { + prompted = true + return importAuthPromptResponse{}, nil + }, + ) + if err != nil { + t.Fatalf("ensureCanvasImportPippitAuth() error = %v", err) + } + if auth.probes != 1 || prompted || len(auth.setValues) != 0 { + t.Fatalf("probes/prompted/sets = %d/%v/%v, want 1/false/none", auth.probes, prompted, auth.setValues) + } +} + +func TestEnsureCanvasImportPippitAuthMissingKeyIsSideEffectFreeWhenNonInteractive(t *testing.T) { + auth := &fakeImportAuthAPI{} + + err := ensureCanvasImportPippitAuth(context.Background(), auth, false, nil) + if err == nil || !strings.Contains(err.Error(), "未找到小云雀 Access Key") { + t.Fatalf("ensureCanvasImportPippitAuth() error = %v, want missing-key guidance", err) + } + if auth.probes != 0 || len(auth.setValues) != 0 { + t.Fatalf("probes/sets = %d/%v, want no side effects", auth.probes, auth.setValues) + } +} + +func TestEnsureCanvasImportPippitAuthPromptsForMissingKeyWithoutProbingEmptyInput(t *testing.T) { + auth := &fakeImportAuthAPI{} + responses := []importAuthPromptResponse{ + {Action: importAuthPromptReplace, AccessKey: " "}, + {Action: importAuthPromptReplace, AccessKey: " pasted-key "}, + } + requests := make([]importAuthPromptRequest, 0, len(responses)) + + err := ensureCanvasImportPippitAuth( + context.Background(), + auth, + true, + func(_ context.Context, request importAuthPromptRequest) (importAuthPromptResponse, error) { + requests = append(requests, request) + response := responses[0] + responses = responses[1:] + return response, nil + }, + ) + if err != nil { + t.Fatalf("ensureCanvasImportPippitAuth() error = %v", err) + } + if len(requests) != 2 || requests[0].HasAccessKey || requests[1].HasAccessKey { + t.Fatalf("prompt requests = %#v, want two missing-key prompts", requests) + } + if !strings.Contains(requests[1].Failure, "不能为空") { + t.Fatalf("second prompt failure = %q, want empty-key guidance", requests[1].Failure) + } + if auth.probes != 1 || strings.Join(auth.setValues, ",") != "pasted-key" { + t.Fatalf("probes/sets = %d/%v, want one verified in-memory update", auth.probes, auth.setValues) + } +} + +func TestEnsureCanvasImportPippitAuthRetriesAndReplacesInvalidKey(t *testing.T) { + auth := &fakeImportAuthAPI{ + accessKey: "invalid-secret-key", + probeErrors: []error{ + errors.New("HTTP 401 invalid-secret-key"), + errors.New("网络暂时不可用"), + nil, + }, + } + responses := []importAuthPromptResponse{ + {Action: importAuthPromptRetry}, + {Action: importAuthPromptReplace, AccessKey: "replacement-key"}, + } + requests := make([]importAuthPromptRequest, 0, len(responses)) + + err := ensureCanvasImportPippitAuth( + context.Background(), + auth, + true, + func(_ context.Context, request importAuthPromptRequest) (importAuthPromptResponse, error) { + requests = append(requests, request) + response := responses[0] + responses = responses[1:] + return response, nil + }, + ) + if err != nil { + t.Fatalf("ensureCanvasImportPippitAuth() error = %v", err) + } + if auth.probes != 3 || strings.Join(auth.setValues, ",") != "replacement-key" { + t.Fatalf("probes/sets = %d/%v, want retry then replacement", auth.probes, auth.setValues) + } + if len(requests) != 2 || !requests[0].HasAccessKey || !requests[1].HasAccessKey { + t.Fatalf("prompt requests = %#v, want failed-key prompts", requests) + } + if strings.Contains(requests[0].Failure, "invalid-secret-key") || !strings.Contains(requests[0].Failure, "HTTP 401") { + t.Fatalf("first prompt failure is not safe, actionable Chinese guidance: %q", requests[0].Failure) + } +} + +func TestEnsureCanvasImportPippitAuthCanCancelAfterProbeFailure(t *testing.T) { + auth := &fakeImportAuthAPI{ + accessKey: "configured-key", + probeErrors: []error{errors.New("HTTP 401")}, + } + + err := ensureCanvasImportPippitAuth( + context.Background(), + auth, + true, + func(context.Context, importAuthPromptRequest) (importAuthPromptResponse, error) { + return importAuthPromptResponse{Action: importAuthPromptCancel}, nil + }, + ) + if !errors.Is(err, errCanvasImportAuthCanceled) { + t.Fatalf("ensureCanvasImportPippitAuth() error = %v, want cancellation", err) + } + if auth.probes != 1 || len(auth.setValues) != 0 { + t.Fatalf("probes/sets = %d/%v, want one read-only probe and no update", auth.probes, auth.setValues) + } +} + +func TestEnsureCanvasImportPippitAuthDoesNotExposePromptErrorText(t *testing.T) { + const candidate = "candidate-key-from-secret-input" + + err := ensureCanvasImportPippitAuth( + context.Background(), + &fakeImportAuthAPI{}, + true, + func(context.Context, importAuthPromptRequest) (importAuthPromptResponse, error) { + return importAuthPromptResponse{}, errors.New("failed after reading " + candidate) + }, + ) + if err == nil || !strings.Contains(err.Error(), "读取小云雀授权选择失败") { + t.Fatalf("ensureCanvasImportPippitAuth() error = %v, want safe prompt failure", err) + } + if strings.Contains(err.Error(), candidate) { + t.Fatalf("ensureCanvasImportPippitAuth() leaked prompt input: %v", err) + } +} + +func TestEnsureCanvasImportPippitAuthRedactsNonInteractiveProbeFailure(t *testing.T) { + const accessKey = "do-not-print-this-key" + auth := &fakeImportAuthAPI{ + accessKey: accessKey, + probeErrors: []error{errors.New("rejected do-not-print-this-key")}, + } + + err := ensureCanvasImportPippitAuth(context.Background(), auth, false, nil) + if err == nil || !strings.Contains(err.Error(), "Access Key 校验失败") { + t.Fatalf("ensureCanvasImportPippitAuth() error = %v, want validation failure", err) + } + if strings.Contains(err.Error(), accessKey) { + t.Fatalf("ensureCanvasImportPippitAuth() leaked Access Key: %v", err) + } +} + +func TestRunnerImportAuthAPIUsesReadOnlyCanvasQueryAndMemoryOnlyKey(t *testing.T) { + var method string + var assetIDs []string + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + method = request.Method + if got := request.Header.Get("Authorization"); got != "Bearer pasted-key" { + t.Errorf("Authorization = %q, want updated in-memory key", got) + } + var body struct { + PippitAssetIDs []string `json:"pippit_asset_ids"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Errorf("Decode() error = %v", err) + } + assetIDs = body.PippitAssetIDs + writer.Header().Set("Content-Type", "application/json") + // The real query endpoint returns an empty data object for a missing + // sentinel ID, so authorization probing must not require data.Assets. + _, _ = writer.Write([]byte(`{"ret":"0","log_id":"probe-log","data":{}}`)) + })) + defer server.Close() + + cfg := &config.Config{BaseURL: server.URL, HTTPTimeout: time.Second} + runner := common.NewRunner(cfg, nil) + runner.Client = common.NewHTTPClient( + cfg.BaseURL, + cfg.HTTPTimeout, + common.NewAccessKeyProviderAuthorizer(func() string { return runner.Config.AccessKey }), + ) + auth := runnerImportAuthAPI{runner: runner} + + if err := auth.SetAccessKey(" pasted-key "); err != nil { + t.Fatalf("SetAccessKey() error = %v", err) + } + if err := auth.Probe(context.Background()); err != nil { + t.Fatalf("Probe() error = %v", err) + } + if method != http.MethodPost || strings.Join(assetIDs, ",") != "9223372036854775807" { + t.Fatalf("probe method/assets = %q/%v, want read-only Canvas query sentinel 9223372036854775807", method, assetIDs) + } + if cfg.AccessKey != "pasted-key" { + t.Fatalf("Config.AccessKey = %q, want trimmed in-memory key", cfg.AccessKey) + } +} + +func TestEnsureCanvasImportPippitAuthHonorsCanceledContextBeforePrompt(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + prompted := false + + err := ensureCanvasImportPippitAuth( + ctx, + &fakeImportAuthAPI{}, + true, + func(context.Context, importAuthPromptRequest) (importAuthPromptResponse, error) { + prompted = true + return importAuthPromptResponse{}, nil + }, + ) + if !errors.Is(err, context.Canceled) || prompted { + t.Fatalf("error/prompted = %v/%v, want canceled before prompt", err, prompted) + } +} + +func TestRedactCanvasImportFinalErrorRemovesCurrentAccessKey(t *testing.T) { + const accessKey = "current-secret-access-key" + auth := &fakeImportAuthAPI{accessKey: accessKey} + err := redactCanvasImportFinalError( + fmt.Errorf("远端失败，响应包含 %s", accessKey), + auth, + ) + if err == nil || strings.Contains(err.Error(), accessKey) || !strings.Contains(err.Error(), "[已隐藏]") { + t.Fatalf("redacted error = %v, want hidden current key", err) + } +} + +func TestCanvasImportPippitAuthFailureRecognizesBusinessRetCode(t *testing.T) { + for _, message := range []string{`query failed: ret=1015`, `query failed: ret="1015"`} { + if !isCanvasImportPippitAuthFailure(errors.New(message)) { + t.Fatalf("isCanvasImportPippitAuthFailure(%q) = false, want true", message) + } + } +} From 30e065e0c34a85fd00e8072987c4a31ffcf848e6 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 22:48:41 +0800 Subject: [PATCH 34/48] feat(canvas): keep imports running through auth recovery Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import.go | 334 +++++++++++++++++++++++++------ cmd/canvas/import_auth_flow.go | 127 ++++++++++++ cmd/canvas/import_media.go | 186 +++++++++-------- cmd/canvas/import_result_i18n.go | 29 +++ 4 files changed, 532 insertions(+), 144 deletions(-) create mode 100644 cmd/canvas/import_auth_flow.go create mode 100644 cmd/canvas/import_result_i18n.go diff --git a/cmd/canvas/import.go b/cmd/canvas/import.go index f49b814..6206be8 100644 --- a/cmd/canvas/import.go +++ b/cmd/canvas/import.go @@ -41,12 +41,18 @@ type importExporter interface { Export(context.Context, string, string, io.Writer) (*libTVExportResult, error) } +type importSourceAuthenticator interface { + Authenticate(context.Context, bool, io.Writer) error +} + type importExecutor interface { Execute(context.Context, canvasplan.Plan, canvasplan.ResolvedMediaSet, canvasplan.ExecuteOptions) (*canvasplan.ExecutionResult, error) Reconcile(context.Context, string, canvasplan.Plan, canvasplan.ResolvedMediaSet) (*canvasplan.ExecutionResult, error) } type importDependencies struct { + pippitAuth importAuthAPI + sourceAuth importSourceAuthenticator exporter importExporter media importMediaAPI executor importExecutor @@ -83,8 +89,11 @@ func (executor runnerImportExecutor) Reconcile( } func newImportDependencies(runner *common.Runner) importDependencies { + libTV := nodeLibTVExporter{} return importDependencies{ - exporter: nodeLibTVExporter{}, + pippitAuth: runnerImportAuthAPI{runner: runner}, + sourceAuth: libTV, + exporter: libTV, media: runnerImportMediaAPI{runner: runner}, executor: runnerImportExecutor{executor: canvasplan.NewExecutor(runner)}, openURL: openBrowserURL, @@ -105,9 +114,9 @@ func newImportCommand( var opts importOptions cmd := &cobra.Command{ Use: "import", - Short: "Import an external project into a personal novel Canvas", - Long: "Import an external project into a personal novel Canvas. " + - "Run without source flags for a guided import; flags remain available for Agent and CI automation.", + Short: "将外部项目导入个人漫剧画布", + Long: "将外部项目导入个人漫剧画布。" + + "不传来源参数时会进入交互式向导；Agent 和 CI 自动化仍可使用完整参数。", Example: " pippit-tool-cli --ppe-env ppe_cli_canvas_ak canvas import", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { @@ -122,11 +131,13 @@ func newImportCommand( } result, err := runCanvasImport(cmd.Context(), prepared, dependencies, stderr, prompts) if result != nil { + localizeCanvasImportResult(result) if writeErr := common.WriteJSON(stdout, result); writeErr != nil { return writeErr } } if err != nil { + err = redactCanvasImportFinalError(err, dependencies.pippitAuth) logCanvasError("canvas import", err, map[string]string{ "provider": strings.TrimSpace(prepared.Provider), "journal": filepath.Base(strings.TrimSpace(prepared.JournalPath)), @@ -139,11 +150,11 @@ func newImportCommand( cmd.SetOut(stdout) cmd.SetErr(stderr) flags := cmd.Flags() - flags.StringVar(&opts.Provider, "from", "", "source provider (currently: libtv)") - flags.StringVar(&opts.SourceURL, "url", "", "source project URL") - flags.BoolVar(&opts.Open, "open", false, "open the verified personal novel Canvas") - flags.BoolVar(&opts.AcceptDegradations, "accept-degradations", false, "accept explicitly reported source conversion degradations") - flags.StringVar(&opts.JournalPath, "journal", "", "durable resume journal path (generated when omitted)") + flags.StringVar(&opts.Provider, "from", "", "导入来源（当前仅支持 libtv）") + flags.StringVar(&opts.SourceURL, "url", "", "来源项目链接") + flags.BoolVar(&opts.Open, "open", false, "导入完成后打开已验证的个人漫剧画布") + flags.BoolVar(&opts.AcceptDegradations, "accept-degradations", false, "接受来源转换中明确报告的能力降级") + flags.StringVar(&opts.JournalPath, "journal", "", "断点续跑记录路径（省略时自动生成）") return cmd } @@ -155,7 +166,7 @@ func runCanvasImport( prompts *importPromptSession, ) (*canvasplan.ExecutionResult, error) { if strings.ToLower(strings.TrimSpace(opts.Provider)) != "libtv" { - return nil, fmt.Errorf("canvas import --from must be libtv") + return nil, fmt.Errorf("canvas import 的 --from 目前必须为 libtv") } sourceURL, err := normalizeLibTVURL(opts.SourceURL) if err != nil { @@ -168,14 +179,14 @@ func runCanvasImport( if explicitJournal != "" { opts.JournalPath = explicitJournal } - bundleRoot, outputDir, err := newImportBundlePath(dependencies.userCacheDir) - if err != nil { + if err := preflightCanvasImportAuth(ctx, dependencies, prompts, stderr); err != nil { return nil, err } - fmt.Fprintln(stderr, "Phase export: exporting the LibTV canvas and its media...") - exported, err := dependencies.exporter.Export(ctx, sourceURL, outputDir, stderr) + bundleRoot, outputDir, exported, err := exportLibTVCanvasWithRetry( + ctx, sourceURL, dependencies, prompts, stderr, + ) if err != nil { - return nil, fmt.Errorf("export LibTV canvas: %w", err) + return nil, err } if err := validateExportLocation(exported, outputDir); err != nil { _ = removeOwnedBundle(outputDir, bundleRoot) @@ -196,12 +207,12 @@ func runCanvasImport( } else if prompts != nil && !opts.AcceptDegradationsExplicit { fmt.Fprintf( stderr, - "Warning: LibTV export contains %d known nonfatal degradation(s), such as empty-media placeholders or semantic downgrades; continuing the interactive import. The final JSON records degradation_count.\n", + "提示：LibTV 导出结果包含 %d 项已知的非致命能力降级，例如空素材占位或语义降级；交互式导入将自动继续，最终 JSON 会记录 degradation_count。\n", len(plan.Degradations), ) } else { return nil, fmt.Errorf( - "LibTV export reports %d explicit degradation(s); inspect %s (plan: %s), then rerun with --accept-degradations", + "LibTV 导出结果包含 %d 项明确的能力降级；请检查 %s（计划文件：%s），确认后使用 --accept-degradations 重新运行", len(plan.Degradations), outputDir, exported.PlanPath, ) } @@ -229,12 +240,12 @@ func runCanvasImport( } if err := canvasplan.PreflightJournalPath(journalPath); err != nil { _ = removeOwnedBundle(outputDir, bundleRoot) - return nil, fmt.Errorf("preflight resolved Canvas import journal before media upload: %w", err) + return nil, fmt.Errorf("上传素材前检查画布导入断点记录失败：%w", err) } - fmt.Fprintf(stderr, "Resume journal: %s\n", journalPath) + fmt.Fprintf(stderr, "断点续跑记录：%s\n", journalPath) checkpointPath := journalPath + ".media.json" - fmt.Fprintf(stderr, "Phase media: resolving %d exported media file(s)...\n", len(media)) - resolved, err := resolveImportMedia(ctx, mediaResolutionOptions{ + fmt.Fprintf(stderr, "阶段：正在处理 %d 个导出素材…\n", len(media)) + mediaOptions := mediaResolutionOptions{ Plan: plan, Media: media, Target: target, @@ -244,31 +255,181 @@ func runCanvasImport( CheckpointPath: checkpointPath, PollInterval: dependencies.mediaPoll, WaitTimeout: dependencies.mediaTimeout, - }, dependencies.media, stderr) - if err != nil { - return nil, err + } + var resolved canvasplan.ResolvedMediaSet + for { + resolved, err = resolveImportMedia(ctx, mediaOptions, dependencies.media, stderr) + if err == nil { + break + } + if prompts != nil && errors.Is(err, errCanvasImportMediaStillProcessing) { + fmt.Fprintln(stderr, "小云雀仍在处理已上传素材；持久化 ID 已保存，CLI 将继续只读查询，不会重复上传。") + if waitErr := waitCanvasImportRetry(ctx, dependencies.mediaPoll); waitErr != nil { + return nil, waitErr + } + continue + } + if prompts == nil || !isCanvasImportPippitAuthFailure(err) { + return nil, err + } + fmt.Fprintln(stderr, "小云雀授权在素材处理期间失效；已保留安全断点，不会重复上传，重新授权后将继续。") + if authErr := ensureCanvasImportPippitAuth( + ctx, dependencies.pippitAuth, true, prompts.promptPippitAuth, + ); authErr != nil { + return nil, authErr + } } plan, err = canonicalizeImportPlanMedia(plan, target, journalPath, checkpointPath) if err != nil { - return nil, fmt.Errorf("canonicalize CanvasPlan media identities: %w", err) + return nil, fmt.Errorf("统一画布计划中的素材标识失败：%w", err) + } + fmt.Fprintln(stderr, "阶段：写入画布前再次确认小云雀授权…") + var pippitPrompt importAuthPrompt + if prompts != nil { + pippitPrompt = prompts.promptPippitAuth + } + if err := ensureCanvasImportPippitAuth( + ctx, dependencies.pippitAuth, prompts != nil, pippitPrompt, + ); err != nil { + return nil, err } result, handled, reconcileErr := reconcileExistingCanvasImport( - ctx, journalPath, plan, resolved, opts, dependencies, stderr, + ctx, journalPath, plan, resolved, opts, dependencies, stderr, prompts, ) if handled { _ = removeOwnedBundle(outputDir, bundleRoot) return result, reconcileErr } - fmt.Fprintln(stderr, "Phase canvas: create/resume, materialize, apply, then verify remote Canvas assets.") - result, executeErr := dependencies.executor.Execute(ctx, plan, resolved, canvasplan.ExecuteOptions{ - JournalPath: journalPath, - }) - if executeErr != nil { - return result, fmt.Errorf("execute CanvasPlan: %w", executeErr) + fmt.Fprintln(stderr, "阶段：正在创建或续跑画布、写入节点与连线，并回读验证远端画布素材…") + for { + result, err = dependencies.executor.Execute(ctx, plan, resolved, canvasplan.ExecuteOptions{ + JournalPath: journalPath, + }) + if err != nil { + if prompts != nil && isCanvasImportPippitAuthFailure(err) && canvasImportStateCanRetryAfterAuth(result) { + fmt.Fprintln(stderr, "小云雀授权在画布处理期间失效；断点已保存，重新授权后将从安全状态继续。") + if authErr := ensureCanvasImportPippitAuth( + ctx, dependencies.pippitAuth, true, prompts.promptPippitAuth, + ); authErr != nil { + return result, authErr + } + continue + } + if prompts != nil && canvasImportStateCanContinueByQuery(result) { + fmt.Fprintln(stderr, "远端画布写入状态暂时无法确认；断点已保存，CLI 将继续只读回查，不会重复提交写入。") + if waitErr := waitCanvasImportRetry(ctx, dependencies.mediaPoll); waitErr != nil { + return result, waitErr + } + continue + } + return result, fmt.Errorf("执行画布计划失败：%w", err) + } + if result != nil && result.State == canvasplan.StateCreatePending { + if prompts != nil && isCanvasImportPippitAuthFailure(errors.New(result.Warning)) { + fmt.Fprintln(stderr, "小云雀授权在等待漫剧画布创建期间失效；创建请求已受理，不会重复创建，重新授权后将继续等待。") + if authErr := ensureCanvasImportPippitAuth( + ctx, dependencies.pippitAuth, true, prompts.promptPippitAuth, + ); authErr != nil { + return result, authErr + } + continue + } + fmt.Fprintln(stderr, "漫剧画布创建请求已受理，正在继续等待服务端完成；不会重复创建项目。") + if waitErr := waitCanvasImportRetry(ctx, dependencies.mediaPoll); waitErr != nil { + return result, waitErr + } + continue + } + break } return finishVerifiedCanvasImport(ctx, result, opts, dependencies, stderr) } +func canvasImportStateCanContinueByQuery(result *canvasplan.ExecutionResult) bool { + if result == nil { + return false + } + return result.State == canvasplan.StateApplyAmbiguous || + result.State == canvasplan.StateVerificationFailed +} + +func canvasImportStateCanRetryAfterAuth(result *canvasplan.ExecutionResult) bool { + if result == nil { + return false + } + switch result.State { + case canvasplan.StateCreatePending, + canvasplan.StateRootReady, + canvasplan.StateAllocationRequested, + canvasplan.StateAllocated, + canvasplan.StateMaterialized, + canvasplan.StateApplyAmbiguous, + canvasplan.StateVerificationFailed: + return true + default: + return false + } +} + +func waitCanvasImportRetry(ctx context.Context, delay time.Duration) error { + if delay <= 0 { + delay = 2 * time.Second + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func exportLibTVCanvasWithRetry( + ctx context.Context, + sourceURL string, + dependencies importDependencies, + prompts *importPromptSession, + stderr io.Writer, +) (string, string, *libTVExportResult, error) { + for { + bundleRoot, outputDir, err := newImportBundlePath(dependencies.userCacheDir) + if err != nil { + return "", "", nil, err + } + fmt.Fprintln(stderr, "阶段：正在导出 LibTV 画布及素材…") + exported, exportErr := dependencies.exporter.Export(ctx, sourceURL, outputDir, stderr) + if exportErr == nil { + return bundleRoot, outputDir, exported, nil + } + _ = removeOwnedBundle(outputDir, bundleRoot) + if ctx.Err() != nil { + return "", "", nil, ctx.Err() + } + if prompts == nil { + return "", "", nil, fmt.Errorf("导出 LibTV 画布失败：%w", exportErr) + } + fmt.Fprintf(stderr, "LibTV 导出未完成：%v\n", exportErr) + choice, promptErr := prompts.askChoice( + "LibTV 导出下一步：", + []importPromptChoice{ + {label: "重新检查授权并重试导出（默认）"}, + {label: "取消导入"}, + }, + 1, + ) + if promptErr != nil { + return "", "", nil, promptErr + } + if choice == 2 { + return "", "", nil, fmt.Errorf("已取消 LibTV 导出和画布导入") + } + if authErr := ensureLibTVImportAuth(ctx, dependencies.sourceAuth, prompts, stderr); authErr != nil { + return "", "", nil, authErr + } + } +} + func reconcileExistingCanvasImport( ctx context.Context, journalPath string, @@ -277,29 +438,56 @@ func reconcileExistingCanvasImport( opts importOptions, dependencies importDependencies, stderr io.Writer, + prompts *importPromptSession, ) (*canvasplan.ExecutionResult, bool, error) { info, err := os.Lstat(journalPath) if os.IsNotExist(err) { return nil, false, nil } if err != nil { - return nil, true, fmt.Errorf("inspect Canvas import journal for reconciliation: %w", err) + return nil, true, fmt.Errorf("检查画布导入断点记录以恢复任务失败：%w", err) } if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { - return nil, true, fmt.Errorf("Canvas import journal must be a regular non-symbolic file") - } - result, reconcileErr := dependencies.executor.Reconcile(ctx, journalPath, plan, resolved) - if errors.Is(reconcileErr, canvasplan.ErrReconcileNotEligible) { - return nil, false, nil + return nil, true, fmt.Errorf("画布导入断点记录必须是普通文件，不能是符号链接") } - if result != nil && !journalOnlyReconcileState(result.State) { - return nil, false, nil + for { + result, reconcileErr := dependencies.executor.Reconcile(ctx, journalPath, plan, resolved) + if errors.Is(reconcileErr, canvasplan.ErrReconcileNotEligible) { + return nil, false, nil + } + if result != nil && !journalOnlyReconcileState(result.State) { + return nil, false, nil + } + if reconcileErr == nil { + result, finishErr := finishVerifiedCanvasImport(ctx, result, opts, dependencies, stderr) + return result, true, finishErr + } + if prompts != nil && isCanvasImportPippitAuthFailure(reconcileErr) { + fmt.Fprintln(stderr, "小云雀授权在恢复画布断点期间失效；重新授权后将继续只读回查，不会重复提交写入。") + if authErr := ensureCanvasImportPippitAuth( + ctx, dependencies.pippitAuth, true, prompts.promptPippitAuth, + ); authErr != nil { + return result, true, authErr + } + continue + } + if prompts != nil && canvasImportReconcileCanContinueByQuery(result, reconcileErr) { + fmt.Fprintln(stderr, "远端画布状态暂时无法确认；CLI 将继续只读回查断点，不会重复提交写入。") + if waitErr := waitCanvasImportRetry(ctx, dependencies.mediaPoll); waitErr != nil { + return result, true, waitErr + } + continue + } + return result, true, fmt.Errorf("在不重复写入的前提下恢复画布导入失败：%w", reconcileErr) } - if reconcileErr != nil { - return result, true, fmt.Errorf("reconcile Canvas import journal without replaying apply: %w", reconcileErr) +} + +func canvasImportReconcileCanContinueByQuery(result *canvasplan.ExecutionResult, err error) bool { + if result == nil || err == nil || !journalOnlyReconcileState(result.State) { + return false } - result, finishErr := finishVerifiedCanvasImport(ctx, result, opts, dependencies, stderr) - return result, true, finishErr + warning := strings.TrimSpace(result.Warning) + return warning != "" && strings.TrimSpace(err.Error()) == warning } func journalOnlyReconcileState(state string) bool { @@ -319,17 +507,17 @@ func finishVerifiedCanvasImport( stderr io.Writer, ) (*canvasplan.ExecutionResult, error) { if !verifiedExecution(result) { - return result, fmt.Errorf("CanvasPlan execution completed without query-back verification") + return result, fmt.Errorf("画布计划已执行，但未通过回读验证") } if opts.Open { if err := validateTrustedCanvasURL(result); err != nil { return result, err } if err := dependencies.openURL(ctx, result.WebURL); err != nil { - fmt.Fprintf(stderr, "Canvas verified, but could not open the browser: %v\n", err) + fmt.Fprintf(stderr, "画布已验证，但无法自动打开浏览器：%v\n", err) } } - fmt.Fprintln(stderr, "Phase canvas: Canvas import verified by query-back.") + fmt.Fprintln(stderr, "阶段：画布导入已通过回读验证。") return result, nil } @@ -337,29 +525,29 @@ func normalizeLibTVURL(value string) (string, error) { raw := strings.TrimSpace(value) parsed, err := url.Parse(raw) if err != nil || parsed.Scheme != "https" || parsed.User != nil { - return "", fmt.Errorf("canvas import --url must be an HTTPS LibTV canvas URL") + return "", fmt.Errorf("canvas import 的 --url 必须是 HTTPS LibTV 画布链接") } host := strings.ToLower(parsed.Hostname()) if (host != "www.liblib.tv" && host != "liblib.tv") || (parsed.Port() != "" && parsed.Port() != "443") { - return "", fmt.Errorf("canvas import --url host must be www.liblib.tv") + return "", fmt.Errorf("canvas import 的 --url 域名必须是 www.liblib.tv") } if strings.TrimRight(parsed.EscapedPath(), "/") != "/canvas" { - return "", fmt.Errorf("canvas import --url must identify a LibTV /canvas project with projectId") + return "", fmt.Errorf("canvas import 的 --url 必须指向带有 projectId 的 LibTV /canvas 项目") } if parsed.Fragment != "" { - return "", fmt.Errorf("canvas import --url must not contain a fragment") + return "", fmt.Errorf("canvas import 的 --url 不能包含片段标识") } query := parsed.Query() projectIDs := query["projectId"] if len(projectIDs) != 1 || !libTVProjectIDPattern.MatchString(strings.TrimSpace(projectIDs[0])) { - return "", fmt.Errorf("canvas import --url projectId must be a LibTV project UUID") + return "", fmt.Errorf("canvas import 的 --url 中 projectId 必须是 LibTV 项目 UUID") } canonical := &url.URL{Scheme: "https", Host: "www.liblib.tv", Path: "/canvas"} canonicalQuery := url.Values{} canonicalQuery.Set("projectId", strings.ToLower(strings.TrimSpace(projectIDs[0]))) if spaceIDs, exists := query["spaceId"]; exists { if len(spaceIDs) != 1 || !libTVSpaceIDPattern.MatchString(strings.TrimSpace(spaceIDs[0])) { - return "", fmt.Errorf("canvas import --url spaceId must be numeric") + return "", fmt.Errorf("canvas import 的 --url 中 spaceId 必须是数字") } canonicalQuery.Set("spaceId", strings.TrimSpace(spaceIDs[0])) } @@ -377,20 +565,20 @@ func resolveImportJournalPath( if value := strings.TrimSpace(explicit); value != "" { absolute, err := filepath.Abs(value) if err != nil { - return "", fmt.Errorf("resolve canvas import journal: %w", err) + return "", fmt.Errorf("解析画布导入断点记录路径失败：%w", err) } return filepath.Clean(absolute), nil } configDir, err := userConfigDir() if err != nil { - return "", fmt.Errorf("resolve canvas import config directory: %w", err) + return "", fmt.Errorf("解析画布导入配置目录失败：%w", err) } directory := filepath.Join(configDir, "pippit-cli", "canvas-import") if err := os.MkdirAll(directory, 0o700); err != nil { - return "", fmt.Errorf("create canvas import journal directory: %w", err) + return "", fmt.Errorf("创建画布导入断点记录目录失败：%w", err) } if err := os.Chmod(directory, 0o700); err != nil { - return "", fmt.Errorf("secure canvas import journal directory: %w", err) + return "", fmt.Errorf("设置画布导入断点记录目录权限失败：%w", err) } hash := sha256.Sum256([]byte(strings.Join([]string{ target, @@ -429,29 +617,29 @@ func verifiedExecution(result *canvasplan.ExecutionResult) bool { func validateTrustedCanvasURL(result *canvasplan.ExecutionResult) error { if !verifiedExecution(result) { - return fmt.Errorf("refusing to open an unverified Canvas result") + return fmt.Errorf("无法打开尚未验证的画布结果") } parsed, err := url.Parse(strings.TrimSpace(result.WebURL)) if err != nil || parsed.Scheme != "https" || parsed.User != nil || strings.ToLower(parsed.Hostname()) != "xyq.jianying.com" || (parsed.Port() != "" && parsed.Port() != "443") || parsed.Fragment != "" { - return fmt.Errorf("refusing to open untrusted Canvas URL") + return fmt.Errorf("无法打开不受信任的画布链接") } if strings.TrimRight(parsed.EscapedPath(), "/") != "/novel/detail/canvas" { - return fmt.Errorf("refusing to open a non-novel Canvas URL") + return fmt.Errorf("无法打开非漫剧画布链接") } query := parsed.Query() if len(query["projectId"]) != 1 || query.Get("projectId") != result.ProjectID { - return fmt.Errorf("refusing to open a Canvas URL whose project ID does not match the verified result") + return fmt.Errorf("画布链接中的项目 ID 与已验证结果不一致，无法打开") } if canvasIDs := query["canvasId"]; len(canvasIDs) > 1 || (len(canvasIDs) == 1 && canvasIDs[0] != result.RootCanvasID) { - return fmt.Errorf("refusing to open a Canvas URL whose canvas ID does not match the verified result") + return fmt.Errorf("画布链接中的画布 ID 与已验证结果不一致，无法打开") } for _, key := range []string{"overviewPippitAssetId", "overview_pippit_asset_id"} { if overviewIDs := query[key]; len(overviewIDs) > 1 || (len(overviewIDs) == 1 && overviewIDs[0] != result.OverviewPippitAssetID) { - return fmt.Errorf("refusing to open a Canvas URL whose overview ID does not match the verified result") + return fmt.Errorf("画布链接中的总览 ID 与已验证结果不一致，无法打开") } } return nil @@ -467,8 +655,26 @@ func openBrowserURL(ctx context.Context, value string) error { default: command = exec.CommandContext(ctx, "xdg-open", value) } + command.Env = sanitizedCanvasOpenEnv(os.Environ()) if err := command.Run(); err != nil { - return fmt.Errorf("open Canvas URL: %w", err) + return fmt.Errorf("打开画布链接失败：%w", err) } return nil } + +func sanitizedCanvasOpenEnv(environ []string) []string { + result := make([]string, 0, len(environ)) + for _, entry := range environ { + key, _, found := strings.Cut(entry, "=") + if !found { + continue + } + switch strings.ToUpper(strings.TrimSpace(key)) { + case "XYQ_ACCESS_KEY", "PIPPIT_ACCESS_KEY", "PIPPIT_AK": + continue + default: + result = append(result, entry) + } + } + return result +} diff --git a/cmd/canvas/import_auth_flow.go b/cmd/canvas/import_auth_flow.go new file mode 100644 index 0000000..31146bf --- /dev/null +++ b/cmd/canvas/import_auth_flow.go @@ -0,0 +1,127 @@ +package canvas + +import ( + "context" + "fmt" + "io" + "strings" +) + +const pippitAccessKeySettingsURL = "https://xyq.jianying.com/home?tab_name=home" + +func preflightCanvasImportAuth( + ctx context.Context, + dependencies importDependencies, + prompts *importPromptSession, + stderr io.Writer, +) error { + interactive := prompts != nil + var pippitPrompt importAuthPrompt + if prompts != nil { + pippitPrompt = prompts.promptPippitAuth + } + + fmt.Fprintln(stderr, "阶段：正在检查小云雀授权…") + if err := ensureCanvasImportPippitAuth(ctx, dependencies.pippitAuth, interactive, pippitPrompt); err != nil { + return err + } + fmt.Fprintln(stderr, "小云雀授权校验通过。") + return ensureLibTVImportAuth(ctx, dependencies.sourceAuth, prompts, stderr) +} + +func ensureLibTVImportAuth( + ctx context.Context, + auth importSourceAuthenticator, + prompts *importPromptSession, + stderr io.Writer, +) error { + if auth == nil { + return fmt.Errorf("LibTV 授权检查未配置") + } + interactive := prompts != nil + for { + fmt.Fprintln(stderr, "阶段：正在检查 LibTV 授权…") + if err := auth.Authenticate(ctx, interactive, stderr); err == nil { + fmt.Fprintln(stderr, "LibTV 授权校验通过；现在开始导出，不会在下载中途再补做登录。") + return nil + } else if ctx.Err() != nil { + return ctx.Err() + } else if !interactive || prompts == nil { + return fmt.Errorf("LibTV 授权校验失败，尚未开始下载任何项目或素材：%w", err) + } else { + fmt.Fprintf(stderr, "LibTV 授权未完成：%v\n", err) + } + + choice, err := prompts.askChoice( + "LibTV 授权下一步：", + []importPromptChoice{ + {label: "重新打开浏览器授权（默认）"}, + {label: "取消导入"}, + }, + 1, + ) + if err != nil { + return err + } + if choice == 2 { + return fmt.Errorf("已取消 LibTV 授权和画布导入") + } + } +} + +func (prompts *importPromptSession) promptPippitAuth( + _ context.Context, + request importAuthPromptRequest, +) (importAuthPromptResponse, error) { + if strings.TrimSpace(request.Failure) != "" { + fmt.Fprintf(prompts.stderr, "小云雀授权提示：%s\n", request.Failure) + } + if !request.HasAccessKey { + fmt.Fprintf( + prompts.stderr, + "未检测到小云雀 Access Key。请先在个人设置页创建或查看：%s\n"+ + "随后在下方粘贴；它只保存在当前 CLI 进程内，不会写入配置、日志或断点记录。\n", + pippitAccessKeySettingsURL, + ) + accessKey, eof, err := prompts.readSecret("粘贴小云雀 Access Key：") + if err != nil { + return importAuthPromptResponse{}, err + } + if eof && strings.TrimSpace(accessKey) == "" { + return importAuthPromptResponse{Action: importAuthPromptCancel}, nil + } + return importAuthPromptResponse{Action: importAuthPromptReplace, AccessKey: accessKey}, nil + } + + defaultChoice := 1 + if strings.Contains(request.Failure, "401") || strings.Contains(request.Failure, "403") { + defaultChoice = 2 + } + choice, err := prompts.askChoice( + "小云雀授权下一步：", + []importPromptChoice{ + {label: "重新校验当前 Access Key"}, + {label: "粘贴新的 Access Key"}, + {label: "取消导入"}, + }, + defaultChoice, + ) + if err != nil { + return importAuthPromptResponse{}, err + } + switch choice { + case 1: + return importAuthPromptResponse{Action: importAuthPromptRetry}, nil + case 2: + accessKey, eof, readErr := prompts.readSecret("粘贴新的小云雀 Access Key：") + if readErr != nil { + return importAuthPromptResponse{}, readErr + } + if eof && strings.TrimSpace(accessKey) == "" { + return importAuthPromptResponse{Action: importAuthPromptCancel}, nil + } + return importAuthPromptResponse{Action: importAuthPromptReplace, AccessKey: accessKey}, nil + default: + return importAuthPromptResponse{Action: importAuthPromptCancel}, nil + } +} diff --git a/cmd/canvas/import_media.go b/cmd/canvas/import_media.go index 6afbc6a..a043da1 100644 --- a/cmd/canvas/import_media.go +++ b/cmd/canvas/import_media.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -18,6 +19,8 @@ import ( "github.com/Pippit-dev/pippit-cli/internal/common" ) +var errCanvasImportMediaStillProcessing = errors.New("小云雀素材仍在处理中") + const ( mediaCheckpointSchema = "pippit-canvas-import-media/0.1" mediaStatusReady = "ready" @@ -47,15 +50,15 @@ type runnerImportMediaAPI struct { func (api runnerImportMediaAPI) Upload(ctx context.Context, media validatedImportMedia) (*canvascore.UploadResult, error) { file, identity, err := openInspectedImportMediaFile(media.LocalPath) if err != nil { - return nil, fmt.Errorf("verify canvas import media immediately before upload: %w", err) + return nil, fmt.Errorf("上传前再次验证画布导入素材失败：%w", err) } defer file.Close() if identity.RawSHA256 != media.SHA256 || identity.ContentFingerprint != media.ContentFingerprint || identity.ByteSize != media.ByteSize { - return nil, fmt.Errorf("canvas import media changed before upload dispatch") + return nil, fmt.Errorf("画布导入素材在发起上传前发生了变化") } if _, err := file.Seek(0, io.SeekStart); err != nil { - return nil, fmt.Errorf("rewind verified canvas import media before upload: %w", err) + return nil, fmt.Errorf("上传前重置已验证素材的读取位置失败：%w", err) } return canvascore.Upload(ctx, canvascore.UploadOptions{ FileName: media.FileName, @@ -81,10 +84,10 @@ func (api runnerImportMediaAPI) PreflightUpload(ctx context.Context) error { return err } if api.runner == nil || api.runner.Config == nil { - return fmt.Errorf("canvas media uploader is not configured") + return fmt.Errorf("画布素材上传器尚未配置") } if strings.TrimSpace(api.runner.Config.AccessKey) == "" { - return fmt.Errorf("XYQ_ACCESS_KEY 缺失; authenticate the Pippit CLI before importing media") + return fmt.Errorf("缺少 XYQ_ACCESS_KEY；请先完成小云雀 CLI 授权，再导入素材") } return nil } @@ -135,21 +138,21 @@ func readAndValidateExportMedia(bundleDir string, plan canvasplan.Plan) ([]valid result := make([]validatedImportMedia, 0, len(plan.RequiredMedia)) for _, requirement := range plan.RequiredMedia { if requirement.LocalPath == "" || requirement.URL != "" { - return nil, fmt.Errorf("LibTV CanvasPlan media %q must use a local bundle path", requirement.LogicalID) + return nil, fmt.Errorf("LibTV 画布计划中的素材 %q 必须使用本地导出包路径", requirement.LogicalID) } localPath := filepath.Join(bundleDir, filepath.FromSlash(requirement.LocalPath)) if err := requireFileWithinBundle(localPath, bundleDir); err != nil { - return nil, fmt.Errorf("invalid LibTV media %q: %w", requirement.LogicalID, err) + return nil, fmt.Errorf("LibTV 素材 %q 无效：%w", requirement.LogicalID, err) } identity, err := inspectImportMediaFile(localPath) if err != nil { - return nil, fmt.Errorf("inspect LibTV media %q: %w", requirement.LogicalID, err) + return nil, fmt.Errorf("检查 LibTV 素材 %q 失败：%w", requirement.LogicalID, err) } if requirement.Metadata.ByteSize == nil || identity.ByteSize != *requirement.Metadata.ByteSize { - return nil, fmt.Errorf("LibTV media %q byte size does not match CanvasPlan", requirement.LogicalID) + return nil, fmt.Errorf("LibTV 素材 %q 的文件大小与画布计划不一致", requirement.LogicalID) } if identity.RawSHA256 != requirement.SHA256 { - return nil, fmt.Errorf("LibTV media %q SHA-256 does not match CanvasPlan", requirement.LogicalID) + return nil, fmt.Errorf("LibTV 素材 %q 的 SHA-256 与画布计划不一致", requirement.LogicalID) } result = append(result, validatedImportMedia{ LogicalID: requirement.LogicalID, @@ -176,7 +179,7 @@ func resolveImportMedia( } defer func() { if err := lock.release(); err != nil { - fmt.Fprintf(stderr, "Could not release canvas import media checkpoint lock: %v\n", err) + fmt.Fprintf(stderr, "提示：无法释放画布导入素材断点锁：%v\n", err) } }() @@ -196,11 +199,11 @@ func resolveImportMedia( } if migrated { if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { - return canvasplan.ResolvedMediaSet{}, fmt.Errorf("save migrated canvas import media checkpoint: %w", err) + return canvasplan.ResolvedMediaSet{}, fmt.Errorf("保存迁移后的画布导入素材断点记录失败：%w", err) } } if len(opts.Media) == 0 { - reportImportMediaProgress(stderr, 0, 0, "complete", validatedImportMedia{FileName: "(none)"}) + reportImportMediaProgress(stderr, 0, 0, "complete", validatedImportMedia{FileName: "（无）"}) } queriedReadyAssetIDs := make(map[string]struct{}) for index, media := range opts.Media { @@ -209,28 +212,28 @@ func resolveImportMedia( switch existing.Status { case mediaStatusBlocked: return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "media upload %q is blocked after an unknown outcome; inspect %s and do not retry the bytes blindly", + "素材 %q 的上传结果未知，当前处于 blocked 状态；请检查 %s，不要直接重复上传", media.LogicalID, opts.CheckpointPath, ) case mediaStatusUploadRequested: existing.Status = mediaStatusBlockedInterruption - existing.LastError = "the previous process stopped after persisting upload-requested and before a durable response was checkpointed" + existing.LastError = "上次进程在记录 upload-requested 后、持久化上传结果前中断" if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, *existing); err != nil { return canvasplan.ResolvedMediaSet{}, err } return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "media upload %q was interrupted with an unknown outcome; checkpointed as blocked-on-interruption in %s and will not be uploaded again automatically", + "素材 %q 的上传曾被中断且结果未知；已在 %s 中记录为 blocked-on-interruption，后续不会自动重复上传", media.LogicalID, opts.CheckpointPath, ) case mediaStatusBlockedInterruption: return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "media upload %q is blocked-on-interruption after an unknown outcome; inspect %s and do not retry the bytes blindly", + "素材 %q 的上传结果未知，当前处于 blocked-on-interruption 状态；请检查 %s，不要直接重复上传", media.LogicalID, opts.CheckpointPath, ) case mediaStatusProcessing: if err := waitForImportMediaReady(ctx, opts, api, stderr, index, media, existing.PippitAssetID); err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "wait for previous media upload %q; durable IDs remain in %s: %w", + "等待之前上传的素材 %q 就绪失败；持久化素材 ID 仍保存在 %s：%w", media.LogicalID, opts.CheckpointPath, err, ) } @@ -247,20 +250,20 @@ func resolveImportMedia( ready, err := api.Query(ctx, existing.PippitAssetID) if err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "previously uploaded media %q is unavailable to the current Pippit account; refusing checkpoint reuse: %w", + "当前小云雀账号无法读取之前上传的素材 %q，因此不会复用断点记录：%w", media.LogicalID, err, ) } if !ready { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "previously uploaded media %q is not visible to the current Pippit account; refusing checkpoint reuse", + "当前小云雀账号不可见之前上传的素材 %q，因此不会复用断点记录", media.LogicalID, ) } queriedReadyAssetIDs[existing.PippitAssetID] = struct{}{} } default: - return canvasplan.ResolvedMediaSet{}, fmt.Errorf("media checkpoint %q has invalid status %q", media.LogicalID, existing.Status) + return canvasplan.ResolvedMediaSet{}, fmt.Errorf("素材 %q 的断点记录状态 %q 无效", media.LogicalID, existing.Status) } reportImportMediaProgress(stderr, index+1, len(opts.Media), action, media) continue @@ -271,13 +274,13 @@ func resolveImportMedia( ready, err := api.Query(ctx, duplicate.PippitAssetID) if err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "deduplicated media %q is unavailable to the current Pippit account; refusing checkpoint reuse: %w", + "当前小云雀账号无法读取已去重素材 %q，因此不会复用断点记录：%w", media.LogicalID, err, ) } if !ready { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "deduplicated media %q is not visible to the current Pippit account; refusing checkpoint reuse", + "当前小云雀账号不可见已去重素材 %q，因此不会复用断点记录", media.LogicalID, ) } @@ -303,7 +306,7 @@ func resolveImportMedia( if preflighter, ok := api.(importMediaPreflighter); ok { if err := preflighter.PreflightUpload(ctx); err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "Pippit authentication failed before media upload %q was requested: %w", + "请求上传素材 %q 前，小云雀授权检查失败：%w", media.LogicalID, err, ) } @@ -315,28 +318,28 @@ func resolveImportMedia( ContentFingerprint: media.ContentFingerprint, CanonicalByteSize: media.ByteSize, Status: mediaStatusUploadRequested, - LastError: "upload request is about to be dispatched; interruption requires manual outcome confirmation", + LastError: "即将发起上传请求；若此时中断，需要人工确认上传结果", } if err := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "persist upload-requested checkpoint for media %q: %w", + "保存素材 %q 的 upload-requested 断点记录失败：%w", media.LogicalID, err, ) } entries[media.LogicalID] = &entry reportImportMediaProgress(stderr, index, len(opts.Media), "uploading", media) uploaded, uploadErr := api.Upload(ctx, media) - if uploadErr != nil && strings.Contains(uploadErr.Error(), "XYQ_ACCESS_KEY 缺失") { + if uploadErr != nil && isCanvasImportPippitAuthFailure(uploadErr) { if checkpointErr := removeAndSaveMediaEntry(opts.CheckpointPath, checkpoint, media.LogicalID); checkpointErr != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "Pippit authentication failed before media upload %q was sent, but its upload-requested checkpoint could not be cleared; do not retry blindly: %w", + "发送素材 %q 的上传请求前，小云雀授权失败，且无法清理 upload-requested 断点记录；请勿直接重试：%w", media.LogicalID, checkpointErr, ) } delete(entries, media.LogicalID) return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "Pippit authentication failed before media upload %q was sent: %w", - media.LogicalID, uploadErr, + "%w：发送素材 %q 的上传请求被小云雀明确拒绝，未产生远端写入", + errCanvasImportReauthenticationRequired, media.LogicalID, ) } entry.LastError = "" @@ -346,29 +349,29 @@ func resolveImportMedia( } if uploadErr != nil || uploaded == nil { entry.Status = mediaStatusBlocked - entry.LastError = errorText(uploadErr, "upload returned no result") + entry.LastError = errorText(uploadErr, "上传未返回结果") if checkpointErr := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); checkpointErr != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "media upload %q has an unknown outcome and its blocked checkpoint could not be saved; do not retry the bytes blindly: %w", + "素材 %q 的上传结果未知，且无法保存 blocked 断点记录；请勿直接重复上传：%w", media.LogicalID, checkpointErr, ) } return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "media upload %q has an unknown outcome; checkpointed as blocked in %s: %s", - media.LogicalID, opts.CheckpointPath, errorText(uploadErr, "upload returned no result"), + "素材 %q 的上传结果未知；已在 %s 中记录为 blocked：%s", + media.LogicalID, opts.CheckpointPath, errorText(uploadErr, "上传未返回结果"), ) } if entry.AssetID == "" || entry.PippitAssetID == "" { entry.Status = mediaStatusBlocked - entry.LastError = "upload response omitted durable asset IDs" + entry.LastError = "上传响应缺少可持久化的素材 ID" if checkpointErr := replaceAndSaveMediaEntry(opts.CheckpointPath, checkpoint, entry); checkpointErr != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "media upload %q omitted durable IDs and its blocked checkpoint could not be saved; do not upload again blindly: %w", + "素材 %q 的上传响应缺少可持久化 ID，且无法保存 blocked 断点记录；请勿直接重复上传：%w", media.LogicalID, checkpointErr, ) } return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "media upload %q omitted durable asset IDs; checkpointed as blocked in %s", + "素材 %q 的上传响应缺少可持久化 ID；已在 %s 中记录为 blocked", media.LogicalID, opts.CheckpointPath, ) } @@ -380,7 +383,7 @@ func resolveImportMedia( } if err := waitForImportMediaReady(ctx, opts, api, stderr, index, media, entry.PippitAssetID); err != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( - "wait for media upload %q; durable IDs remain in %s: %w", + "等待素材 %q 上传就绪失败；持久化素材 ID 仍保存在 %s：%w", media.LogicalID, opts.CheckpointPath, err, ) } @@ -443,7 +446,7 @@ func waitForImportMediaReady( return importMediaWaitError(waitCtx.Err(), pippitAssetID, waitTimeout) } return fmt.Errorf( - "query processing Pippit asset %q failed; this is a read/authentication error, not a processing signal: %w", + "查询处理中小云雀素材 %q 失败；这是读取或授权错误，不代表素材仍在处理中：%w", pippitAssetID, err, ) @@ -463,9 +466,9 @@ func waitForImportMediaReady( func importMediaWaitError(err error, pippitAssetID string, timeout time.Duration) error { if err == context.DeadlineExceeded { - return fmt.Errorf("Pippit asset %q was not visible within %s; it will only be queried on the next run and will not be uploaded again", pippitAssetID, timeout) + return fmt.Errorf("%w：素材 %q 在 %s 内仍不可见；只会继续查询，不会重复上传", errCanvasImportMediaStillProcessing, pippitAssetID, timeout) } - return fmt.Errorf("wait for Pippit asset %q canceled; it will not be uploaded again: %w", pippitAssetID, err) + return fmt.Errorf("等待小云雀素材 %q 就绪已取消；不会重复上传：%w", pippitAssetID, err) } func reportImportMediaProgress( @@ -485,32 +488,55 @@ func reportImportMediaProgress( } fmt.Fprintf( stderr, - "Media progress: processed=%d/%d remaining=%d action=%s file=%q\n", + "素材进度：已处理=%d/%d，剩余=%d，状态=%s，文件=%q\n", processed, total, remaining, - action, + importMediaProgressAction(action), fileName, ) } +func importMediaProgressAction(action string) string { + switch action { + case "complete": + return "完成" + case "reused": + return "已复用" + case "queried": + return "已确认" + case "checking": + return "正在检查" + case "uploading": + return "正在上传" + case "uploaded": + return "已上传" + case "processing": + return "正在处理" + case "waiting": + return "等待处理中" + default: + return action + } +} + func loadMediaCheckpoint(opts mediaResolutionOptions) (*mediaCheckpoint, error) { if info, lstatErr := os.Lstat(opts.CheckpointPath); lstatErr == nil { if info.Mode()&os.ModeSymlink != 0 { - return nil, fmt.Errorf("canvas import media checkpoint must not be a symbolic link") + return nil, fmt.Errorf("画布导入素材断点记录不能是符号链接") } if info.Size() > maxMediaCheckpointBytes { - return nil, fmt.Errorf("canvas import media checkpoint exceeds %d bytes", maxMediaCheckpointBytes) + return nil, fmt.Errorf("画布导入素材断点记录超过 %d 字节", maxMediaCheckpointBytes) } } else if !os.IsNotExist(lstatErr) { - return nil, fmt.Errorf("inspect canvas import media checkpoint: %w", lstatErr) + return nil, fmt.Errorf("检查画布导入素材断点记录失败：%w", lstatErr) } file, err := os.Open(opts.CheckpointPath) if os.IsNotExist(err) { if _, journalErr := os.Stat(opts.CanvasJournalPath); journalErr == nil { - return nil, fmt.Errorf("canvas journal exists but its media checkpoint is missing; refusing to upload again: %s", opts.CheckpointPath) + return nil, fmt.Errorf("画布断点记录已存在，但素材断点记录缺失；为避免重复上传，已停止处理：%s", opts.CheckpointPath) } else if !os.IsNotExist(journalErr) { - return nil, fmt.Errorf("inspect canvas import journal before media upload: %w", journalErr) + return nil, fmt.Errorf("上传素材前检查画布导入断点记录失败：%w", journalErr) } checkpoint := &mediaCheckpoint{ Schema: mediaCheckpointSchema, @@ -524,23 +550,23 @@ func loadMediaCheckpoint(opts mediaResolutionOptions) (*mediaCheckpoint, error) return checkpoint, nil } if err != nil { - return nil, fmt.Errorf("open canvas import media checkpoint: %w", err) + return nil, fmt.Errorf("打开画布导入素材断点记录失败：%w", err) } defer file.Close() decoder := json.NewDecoder(io.LimitReader(file, maxMediaCheckpointBytes+1)) decoder.DisallowUnknownFields() var checkpoint mediaCheckpoint if err := decoder.Decode(&checkpoint); err != nil { - return nil, fmt.Errorf("decode canvas import media checkpoint: %w", err) + return nil, fmt.Errorf("解析画布导入素材断点记录失败：%w", err) } if err := ensureImportJSONEOF(decoder); err != nil { - return nil, fmt.Errorf("decode canvas import media checkpoint: %w", err) + return nil, fmt.Errorf("解析画布导入素材断点记录失败：%w", err) } if checkpoint.Schema != mediaCheckpointSchema || checkpoint.Source != opts.Plan.Source || checkpoint.Target != opts.Target { - return nil, fmt.Errorf("canvas import media checkpoint does not match this source and target") + return nil, fmt.Errorf("画布导入素材断点记录与当前来源或目标不匹配") } if err := os.Chmod(opts.CheckpointPath, 0o600); err != nil { - return nil, fmt.Errorf("secure canvas import media checkpoint: %w", err) + return nil, fmt.Errorf("设置画布导入素材断点记录权限失败：%w", err) } return &checkpoint, nil } @@ -559,28 +585,28 @@ func validateCheckpointEntries( entry := &checkpoint.Entries[index] item, ok := expected[entry.LogicalID] if !ok || item.MediaType != entry.MediaType { - return nil, false, fmt.Errorf("canvas import media changed after checkpoint creation") + return nil, false, fmt.Errorf("画布导入素材在断点记录创建后发生了变化") } if !validRawMediaSHA256(entry.SHA256) || !validImportMediaContentFingerprint(item.ContentFingerprint) { - return nil, false, fmt.Errorf("canvas import media checkpoint entry %q has an invalid fingerprint", entry.LogicalID) + return nil, false, fmt.Errorf("画布导入素材 %q 的断点指纹无效", entry.LogicalID) } if item.SHA256 != entry.SHA256 { if item.MediaType != "image" { - return nil, false, fmt.Errorf("canvas import media changed after checkpoint creation") + return nil, false, fmt.Errorf("画布导入素材在断点记录创建后发生了变化") } if entry.ContentFingerprint == "" || entry.CanonicalByteSize == 0 { previousIdentity, err := findLegacyCheckpointImageIdentity(checkpoint, opts, *entry) if err != nil { - return nil, false, fmt.Errorf("verify changed checkpoint image %q: %w", entry.LogicalID, err) + return nil, false, fmt.Errorf("验证断点记录中已变化的图片 %q 失败：%w", entry.LogicalID, err) } if !validImportMediaContentFingerprint(previousIdentity.ContentFingerprint) { - return nil, false, fmt.Errorf("canvas import media checkpoint image %q has an invalid content fingerprint", entry.LogicalID) + return nil, false, fmt.Errorf("画布导入图片 %q 的断点内容指纹无效", entry.LogicalID) } if entry.ContentFingerprint != "" && entry.ContentFingerprint != previousIdentity.ContentFingerprint { - return nil, false, fmt.Errorf("canvas import media checkpoint image %q content fingerprint does not match its retained bundle", entry.LogicalID) + return nil, false, fmt.Errorf("画布导入图片 %q 的断点内容指纹与保留的导出包不一致", entry.LogicalID) } if entry.CanonicalByteSize != 0 && entry.CanonicalByteSize != previousIdentity.ByteSize { - return nil, false, fmt.Errorf("canvas import media checkpoint image %q byte size does not match its retained bundle", entry.LogicalID) + return nil, false, fmt.Errorf("画布导入图片 %q 的断点文件大小与保留的导出包不一致", entry.LogicalID) } if entry.ContentFingerprint == "" { entry.ContentFingerprint = previousIdentity.ContentFingerprint @@ -592,19 +618,19 @@ func validateCheckpointEntries( } } if !validImportMediaContentFingerprint(entry.ContentFingerprint) || entry.ContentFingerprint != item.ContentFingerprint { - return nil, false, fmt.Errorf("canvas import image content changed after checkpoint creation") + return nil, false, fmt.Errorf("画布导入图片内容在断点记录创建后发生了变化") } } else { if entry.ContentFingerprint != "" && (!validImportMediaContentFingerprint(entry.ContentFingerprint) || entry.ContentFingerprint != item.ContentFingerprint) { - return nil, false, fmt.Errorf("canvas import media content changed after checkpoint creation") + return nil, false, fmt.Errorf("画布导入素材内容在断点记录创建后发生了变化") } if entry.ContentFingerprint == "" { entry.ContentFingerprint = item.ContentFingerprint migrated = true } if entry.CanonicalByteSize != 0 && entry.CanonicalByteSize != item.ByteSize { - return nil, false, fmt.Errorf("canvas import media byte size changed after checkpoint creation") + return nil, false, fmt.Errorf("画布导入素材大小在断点记录创建后发生了变化") } if entry.CanonicalByteSize == 0 { entry.CanonicalByteSize = item.ByteSize @@ -612,11 +638,11 @@ func validateCheckpointEntries( } } if _, duplicate := entries[entry.LogicalID]; duplicate { - return nil, false, fmt.Errorf("canvas import media checkpoint contains duplicate logical ID %q", entry.LogicalID) + return nil, false, fmt.Errorf("画布导入素材断点记录包含重复的逻辑 ID %q", entry.LogicalID) } if (entry.Status == mediaStatusReady || entry.Status == mediaStatusProcessing) && (strings.TrimSpace(entry.AssetID) == "" || strings.TrimSpace(entry.PippitAssetID) == "") { - return nil, false, fmt.Errorf("canvas import media checkpoint entry %q has no durable IDs", entry.LogicalID) + return nil, false, fmt.Errorf("画布导入素材 %q 的断点记录缺少可持久化 ID", entry.LogicalID) } entries[entry.LogicalID] = entry } @@ -649,12 +675,12 @@ func canonicalizeImportPlanMedia( CheckpointPath: checkpointPath, }) if err != nil { - return canvasplan.Plan{}, fmt.Errorf("load canonical canvas import media identities: %w", err) + return canvasplan.Plan{}, fmt.Errorf("加载画布导入素材的规范标识失败：%w", err) } entries := make(map[string]mediaCheckpointEntry, len(checkpoint.Entries)) for _, entry := range checkpoint.Entries { if _, exists := entries[entry.LogicalID]; exists { - return canvasplan.Plan{}, fmt.Errorf("canvas import media checkpoint contains duplicate logical ID %q", entry.LogicalID) + return canvasplan.Plan{}, fmt.Errorf("画布导入素材断点记录包含重复的逻辑 ID %q", entry.LogicalID) } entries[entry.LogicalID] = entry } @@ -664,10 +690,10 @@ func canonicalizeImportPlanMedia( requirement := &canonical.RequiredMedia[index] entry, ok := entries[requirement.LogicalID] if !ok || entry.MediaType != requirement.MediaType || entry.Status != mediaStatusReady { - return canvasplan.Plan{}, fmt.Errorf("canvas import media checkpoint has no ready canonical identity for %q", requirement.LogicalID) + return canvasplan.Plan{}, fmt.Errorf("画布导入素材断点记录中没有 %q 的可用规范标识", requirement.LogicalID) } if !validRawMediaSHA256(entry.SHA256) || entry.CanonicalByteSize <= 0 { - return canvasplan.Plan{}, fmt.Errorf("canvas import media checkpoint has an invalid canonical identity for %q", requirement.LogicalID) + return canvasplan.Plan{}, fmt.Errorf("画布导入素材 %q 的规范标识无效", requirement.LogicalID) } requirement.SHA256 = entry.SHA256 byteSize := entry.CanonicalByteSize @@ -717,16 +743,16 @@ func removeAndSaveMediaEntry(path string, checkpoint *mediaCheckpoint, logicalID func saveMediaCheckpoint(path string, checkpoint *mediaCheckpoint) error { payload, err := json.MarshalIndent(checkpoint, "", " ") if err != nil { - return fmt.Errorf("encode canvas import media checkpoint: %w", err) + return fmt.Errorf("编码画布导入素材断点记录失败：%w", err) } payload = append(payload, '\n') directory := filepath.Dir(path) if err := os.MkdirAll(directory, 0o700); err != nil { - return fmt.Errorf("create canvas import media checkpoint directory: %w", err) + return fmt.Errorf("创建画布导入素材断点记录目录失败：%w", err) } temporary, err := os.CreateTemp(directory, ".canvas-import-media-*") if err != nil { - return fmt.Errorf("create temporary canvas import media checkpoint: %w", err) + return fmt.Errorf("创建临时画布导入素材断点记录失败：%w", err) } temporaryPath := temporary.Name() defer func() { @@ -734,22 +760,22 @@ func saveMediaCheckpoint(path string, checkpoint *mediaCheckpoint) error { _ = os.Remove(temporaryPath) }() if err := temporary.Chmod(0o600); err != nil { - return fmt.Errorf("secure temporary canvas import media checkpoint: %w", err) + return fmt.Errorf("设置临时画布导入素材断点记录权限失败：%w", err) } if _, err := temporary.Write(payload); err != nil { - return fmt.Errorf("write canvas import media checkpoint: %w", err) + return fmt.Errorf("写入画布导入素材断点记录失败：%w", err) } if err := temporary.Sync(); err != nil { - return fmt.Errorf("sync canvas import media checkpoint: %w", err) + return fmt.Errorf("同步画布导入素材断点记录失败：%w", err) } if err := temporary.Close(); err != nil { - return fmt.Errorf("close canvas import media checkpoint: %w", err) + return fmt.Errorf("关闭画布导入素材断点记录失败：%w", err) } if err := os.Rename(temporaryPath, path); err != nil { - return fmt.Errorf("replace canvas import media checkpoint: %w", err) + return fmt.Errorf("替换画布导入素材断点记录失败：%w", err) } if err := os.Chmod(path, 0o600); err != nil { - return fmt.Errorf("secure canvas import media checkpoint: %w", err) + return fmt.Errorf("设置画布导入素材断点记录权限失败：%w", err) } return nil } @@ -762,13 +788,13 @@ func cleanupCheckpointBundles( remaining := make([]string, 0, len(checkpoint.BundleDirs)) for _, bundleDir := range checkpoint.BundleDirs { if err := removeOwnedBundle(bundleDir, opts.BundleRoot); err != nil { - fmt.Fprintf(stderr, "Could not remove local LibTV export bundle %s: %v\n", bundleDir, err) + fmt.Fprintf(stderr, "提示：无法删除本地 LibTV 导出包 %s：%v\n", bundleDir, err) remaining = append(remaining, bundleDir) } } checkpoint.BundleDirs = remaining if err := saveMediaCheckpoint(opts.CheckpointPath, checkpoint); err != nil { - fmt.Fprintf(stderr, "Could not update media checkpoint cleanup state: %v\n", err) + fmt.Fprintf(stderr, "提示：无法更新素材断点记录的清理状态：%v\n", err) } } diff --git a/cmd/canvas/import_result_i18n.go b/cmd/canvas/import_result_i18n.go new file mode 100644 index 0000000..0141936 --- /dev/null +++ b/cmd/canvas/import_result_i18n.go @@ -0,0 +1,29 @@ +package canvas + +import "github.com/Pippit-dev/pippit-cli/internal/canvasplan" + +// localizeCanvasImportResult keeps the provider-neutral executor contract +// unchanged while making the import command's user-facing warning Chinese. +func localizeCanvasImportResult(result *canvasplan.ExecutionResult) { + if result == nil || result.Warning == "" { + return + } + if result.Verification != nil && result.Verification.Verified && result.Verification.RecoveredFromQuery { + result.Warning = "画布写入响应状态不明确，但已通过精确回读找回并校验全部资产；CLI 未重复提交写入。" + return + } + switch result.State { + case canvasplan.StateVerified: + result.Warning = "画布已通过历史验证；当前授权仍可访问，CLI 未覆盖用户后续编辑，也未重复提交写入。" + case canvasplan.StateCreatePending: + result.Warning = "漫剧画布创建请求已受理，服务端仍在处理中；断点已保存，续跑不会重复创建项目。" + case canvasplan.StateCreateAmbiguous: + result.Warning = "漫剧画布创建结果暂不明确；已保存安全断点，请使用同一断点恢复，切勿重新创建。" + case canvasplan.StateApplyAmbiguous: + result.Warning = "画布写入结果暂不明确；已保存安全断点，恢复时只会回读核对，不会盲目重放写入。" + case canvasplan.StateVerificationFailed: + result.Warning = "画布写入后的回读验证尚未完成；已保存安全断点，CLI 不会盲目重放写入。" + default: + result.Warning = "画布导入已保存安全断点；请排除当前错误后使用同一断点继续。" + } +} From 0b936012a80396e21a71bb4e7773082fc76d71f9 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Tue, 11 Aug 2026 22:49:07 +0800 Subject: [PATCH 35/48] test(canvas): cover authenticated resumable imports Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_auth_flow_test.go | 637 ++++++++++++++++++++++++++ cmd/canvas/import_open_test.go | 26 ++ cmd/canvas/import_result_i18n_test.go | 55 +++ cmd/canvas/import_test.go | 117 +++-- 4 files changed, 809 insertions(+), 26 deletions(-) create mode 100644 cmd/canvas/import_auth_flow_test.go create mode 100644 cmd/canvas/import_open_test.go create mode 100644 cmd/canvas/import_result_i18n_test.go diff --git a/cmd/canvas/import_auth_flow_test.go b/cmd/canvas/import_auth_flow_test.go new file mode 100644 index 0000000..57766fc --- /dev/null +++ b/cmd/canvas/import_auth_flow_test.go @@ -0,0 +1,637 @@ +package canvas + +import ( + "bytes" + "context" + "errors" + "io" + "path/filepath" + "strings" + "testing" + "time" + + canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" + "github.com/Pippit-dev/pippit-cli/internal/canvasplan" +) + +type trackingImportAuthAPI struct { + key string + events *[]string + probeErrors []error + setValues []string +} + +func (auth *trackingImportAuthAPI) AccessKey() string { return auth.key } + +func (auth *trackingImportAuthAPI) SetAccessKey(value string) error { + auth.key = strings.TrimSpace(value) + auth.setValues = append(auth.setValues, auth.key) + return nil +} + +func (auth *trackingImportAuthAPI) Probe(context.Context) error { + if auth.events != nil { + *auth.events = append(*auth.events, "pippit-auth") + } + if len(auth.probeErrors) == 0 { + return nil + } + err := auth.probeErrors[0] + auth.probeErrors = auth.probeErrors[1:] + return err +} + +type trackingSourceAuthenticator struct { + events *[]string + errors []error + calls int +} + +func (auth *trackingSourceAuthenticator) Authenticate(context.Context, bool, io.Writer) error { + auth.calls++ + if auth.events != nil { + *auth.events = append(*auth.events, "libtv-auth") + } + if len(auth.errors) == 0 { + return nil + } + err := auth.errors[0] + auth.errors = auth.errors[1:] + return err +} + +type trackingImportExporter struct { + inner *fakeImportExporter + events *[]string + exportErrors []error +} + +type expiringImportMediaAPI struct { + uploads int +} + +func (api *expiringImportMediaAPI) PreflightUpload(context.Context) error { return nil } + +func (api *expiringImportMediaAPI) Upload( + context.Context, + validatedImportMedia, +) (*canvascore.UploadResult, error) { + api.uploads++ + if api.uploads == 1 { + return nil, errors.New("HTTP 401") + } + return &canvascore.UploadResult{ + State: canvascore.StateReady, AssetID: "asset-after-reauth", PippitAssetID: "pippit-after-reauth", + }, nil +} + +func (*expiringImportMediaAPI) Query(context.Context, string) (bool, error) { return true, nil } + +type sequencedImportExecutor struct { + results []*canvasplan.ExecutionResult + errors []error + calls int +} + +type sequencedReconcileExecutor struct { + results []*canvasplan.ExecutionResult + errors []error + calls int +} + +func (*sequencedReconcileExecutor) Execute( + context.Context, + canvasplan.Plan, + canvasplan.ResolvedMediaSet, + canvasplan.ExecuteOptions, +) (*canvasplan.ExecutionResult, error) { + return nil, errors.New("unexpected Execute call during journal reconciliation") +} + +func (executor *sequencedReconcileExecutor) Reconcile( + context.Context, + string, + canvasplan.Plan, + canvasplan.ResolvedMediaSet, +) (*canvasplan.ExecutionResult, error) { + executor.calls++ + if len(executor.results) == 0 { + return nil, errors.New("unexpected extra Reconcile call") + } + result := executor.results[0] + executor.results = executor.results[1:] + if len(executor.errors) == 0 { + return result, nil + } + err := executor.errors[0] + executor.errors = executor.errors[1:] + return result, err +} + +func (executor *sequencedImportExecutor) Execute( + context.Context, + canvasplan.Plan, + canvasplan.ResolvedMediaSet, + canvasplan.ExecuteOptions, +) (*canvasplan.ExecutionResult, error) { + executor.calls++ + if len(executor.results) == 0 { + return nil, errors.New("unexpected extra Execute call") + } + result := executor.results[0] + executor.results = executor.results[1:] + if len(executor.errors) == 0 { + return result, nil + } + err := executor.errors[0] + executor.errors = executor.errors[1:] + return result, err +} + +func (*sequencedImportExecutor) Reconcile( + context.Context, + string, + canvasplan.Plan, + canvasplan.ResolvedMediaSet, +) (*canvasplan.ExecutionResult, error) { + return nil, canvasplan.ErrReconcileNotEligible +} + +func (exporter *trackingImportExporter) Export( + ctx context.Context, + sourceURL string, + outputDir string, + stderr io.Writer, +) (*libTVExportResult, error) { + *exporter.events = append(*exporter.events, "export") + if len(exporter.exportErrors) != 0 { + err := exporter.exportErrors[0] + exporter.exportErrors = exporter.exportErrors[1:] + return nil, err + } + return exporter.inner.Export(ctx, sourceURL, outputDir, stderr) +} + +func TestCanvasImportPreflightsBothAccountsBeforeExport(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + events := []string{} + innerExporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + exporter := &trackingImportExporter{inner: innerExporter, events: &events} + media := &fakeImportMediaAPI{} + executor := &fakeImportExecutor{result: verifiedImportResult()} + deps := testImportDependencies(temp, innerExporter, media, executor) + deps.pippitAuth = &trackingImportAuthAPI{key: "configured-key", events: &events} + deps.sourceAuth = &trackingSourceAuthenticator{events: &events} + deps.exporter = exporter + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, io.Discard, nil) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified { + t.Fatalf("runCanvasImport() result = %#v, want verified", result) + } + if got := strings.Join(events, ","); got != "pippit-auth,libtv-auth,export,pippit-auth" { + t.Fatalf("auth/export order = %q, want both auth probes before export and Pippit recheck before Canvas writes", got) + } +} + +func TestCanvasImportMissingPippitKeyStopsBeforeLibTVOrFilesystemSideEffects(t *testing.T) { + cacheTouched := false + sourceAuth := &trackingSourceAuthenticator{} + exporter := &trackingImportExporter{inner: &fakeImportExporter{}, events: &[]string{}} + deps := importDependencies{ + pippitAuth: &trackingImportAuthAPI{}, + sourceAuth: sourceAuth, + exporter: exporter, + userCacheDir: func() (string, error) { + cacheTouched = true + return t.TempDir(), nil + }, + } + + _, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, io.Discard, nil) + if err == nil || !strings.Contains(err.Error(), "未找到小云雀 Access Key") { + t.Fatalf("runCanvasImport() error = %v, want missing-key guidance", err) + } + if sourceAuth.calls != 0 || len(exporter.inner.urls) != 0 || cacheTouched { + t.Fatalf("source auth/export/cache side effects = %d/%d/%v, want all zero", sourceAuth.calls, len(exporter.inner.urls), cacheTouched) + } +} + +func TestCanvasImportAuthPromptsForPippitKeyThenChecksLibTV(t *testing.T) { + const accessKey = "pasted-secret-access-key" + events := []string{} + pippit := &trackingImportAuthAPI{events: &events} + source := &trackingSourceAuthenticator{events: &events} + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader(accessKey+"\n"), &stderr, false, + ) + + err := preflightCanvasImportAuth(context.Background(), importDependencies{ + pippitAuth: pippit, + sourceAuth: source, + }, prompts, &stderr) + if err != nil { + t.Fatalf("preflightCanvasImportAuth() error = %v", err) + } + if got := strings.Join(events, ","); got != "pippit-auth,libtv-auth" { + t.Fatalf("auth order = %q, want Pippit then LibTV", got) + } + if pippit.key != accessKey || len(pippit.setValues) != 1 { + t.Fatalf("in-memory Access Key = %q / %v", pippit.key, pippit.setValues) + } + if strings.Contains(stderr.String(), accessKey) { + t.Fatalf("stderr leaked pasted Access Key: %q", stderr.String()) + } +} + +func TestCanvasImportAuthReplacesRejectedPippitKeyWithoutLeakingIt(t *testing.T) { + const oldKey = "rejected-secret-key" + const newKey = "replacement-secret-key" + pippit := &trackingImportAuthAPI{ + key: oldKey, + probeErrors: []error{errors.New("HTTP 401 " + oldKey), nil}, + } + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader("2\n"+newKey+"\n"), &stderr, false, + ) + + err := preflightCanvasImportAuth(context.Background(), importDependencies{ + pippitAuth: pippit, + sourceAuth: &trackingSourceAuthenticator{}, + }, prompts, &stderr) + if err != nil { + t.Fatalf("preflightCanvasImportAuth() error = %v", err) + } + if pippit.key != newKey { + t.Fatalf("Access Key = %q, want replacement", pippit.key) + } + if strings.Contains(stderr.String(), oldKey) || strings.Contains(stderr.String(), newKey) { + t.Fatalf("stderr leaked an Access Key: %q", stderr.String()) + } +} + +func TestCanvasImportAuthRetriesRecoverableLibTVFailure(t *testing.T) { + source := &trackingSourceAuthenticator{errors: []error{errors.New("浏览器授权暂时失败"), nil}} + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader("1\n"), &stderr, false, + ) + + err := preflightCanvasImportAuth(context.Background(), importDependencies{ + pippitAuth: &trackingImportAuthAPI{key: "configured-key"}, + sourceAuth: source, + }, prompts, &stderr) + if err != nil { + t.Fatalf("preflightCanvasImportAuth() error = %v", err) + } + if source.calls != 2 { + t.Fatalf("LibTV auth calls = %d, want retry in the same process", source.calls) + } + for _, expected := range []string{"LibTV 授权未完成", "重新打开浏览器授权", "授权校验通过"} { + if !strings.Contains(stderr.String(), expected) { + t.Fatalf("stderr missing %q: %q", expected, stderr.String()) + } + } +} + +func TestCanvasImportRetriesLibTVExportAfterRecheckingAuth(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + events := []string{} + innerExporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + exporter := &trackingImportExporter{ + inner: innerExporter, events: &events, + exportErrors: []error{errors.New("LibTV 网络暂时不可用")}, + } + source := &trackingSourceAuthenticator{events: &events} + deps := testImportDependencies(temp, innerExporter, &fakeImportMediaAPI{}, &fakeImportExecutor{result: verifiedImportResult()}) + deps.pippitAuth = &trackingImportAuthAPI{key: "configured-key", events: &events} + deps.sourceAuth = source + deps.exporter = exporter + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader("1\n"), &stderr, false, + ) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, &stderr, prompts) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified { + t.Fatalf("runCanvasImport() result = %#v, want verified", result) + } + if source.calls != 2 { + t.Fatalf("LibTV auth calls = %d, want preflight plus retry recheck", source.calls) + } + if got := strings.Join(events, ","); got != "pippit-auth,libtv-auth,export,libtv-auth,export,pippit-auth" { + t.Fatalf("events = %q, want auth before each export attempt", got) + } +} + +func TestCanvasImportReauthorizesDuringMediaWithoutBlindUploadReplay(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &expiringImportMediaAPI{} + executor := &fakeImportExecutor{result: verifiedImportResult()} + deps := testImportDependencies(temp, exporter, media, executor) + deps.pippitAuth = &trackingImportAuthAPI{ + key: "expired-key", + probeErrors: []error{ + nil, + errors.New("HTTP 401"), + nil, + nil, + }, + } + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader("2\nreplacement-key\n"), &stderr, false, + ) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, &stderr, prompts) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified { + t.Fatalf("runCanvasImport() result = %#v, want verified", result) + } + if media.uploads != 2 { + t.Fatalf("Upload calls = %d, want one explicitly rejected attempt and one post-auth upload", media.uploads) + } + if strings.Contains(stderr.String(), "expired-key") || strings.Contains(stderr.String(), "replacement-key") { + t.Fatalf("stderr leaked Access Key: %q", stderr.String()) + } +} + +func TestCanvasImportWaitsForAcceptedCreateInSameProcess(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + executor := &sequencedImportExecutor{results: []*canvasplan.ExecutionResult{ + {State: canvasplan.StateCreatePending, JournalPath: filepath.Join(temp, "pending.json")}, + verifiedImportResult(), + }} + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + deps.mediaPoll = time.Millisecond + var stderr bytes.Buffer + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, &stderr, nil) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified || executor.calls != 2 { + t.Fatalf("result/calls = %#v/%d, want same-process pending resume", result, executor.calls) + } + if !strings.Contains(stderr.String(), "不会重复创建项目") { + t.Fatalf("stderr missing safe pending progress: %q", stderr.String()) + } +} + +func TestCanvasImportReauthorizesWhileWaitingForAcceptedCreate(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + executor := &sequencedImportExecutor{results: []*canvasplan.ExecutionResult{ + { + State: canvasplan.StateCreatePending, + JournalPath: filepath.Join(temp, "pending.json"), + Warning: "poll canvas creation: HTTP 401", + }, + verifiedImportResult(), + }} + pippit := &trackingImportAuthAPI{ + key: "expired-key", + probeErrors: []error{ + nil, + nil, + errors.New("HTTP 401"), + nil, + }, + } + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + deps.pippitAuth = pippit + deps.mediaPoll = time.Millisecond + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader("2\nreplacement-key\n"), &stderr, false, + ) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, &stderr, prompts) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified || executor.calls != 2 { + t.Fatalf("result/calls = %#v/%d, want reauthorized same-process create resume", result, executor.calls) + } + if pippit.key != "replacement-key" || !strings.Contains(stderr.String(), "不会重复创建") { + t.Fatalf("key/stderr = %q/%q, want safe create reauthorization", pippit.key, stderr.String()) + } +} + +func TestCanvasImportReauthorizesAmbiguousApplyThenQueriesWithoutReplay(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + executor := &sequencedImportExecutor{ + results: []*canvasplan.ExecutionResult{ + {State: canvasplan.StateApplyAmbiguous, JournalPath: filepath.Join(temp, "ambiguous.json")}, + verifiedImportResult(), + }, + errors: []error{errors.New("canvas apply failed; exact query-back failed: HTTP 401")}, + } + pippit := &trackingImportAuthAPI{ + key: "expired-key", + probeErrors: []error{ + nil, + nil, + errors.New("HTTP 401"), + nil, + }, + } + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + deps.pippitAuth = pippit + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader("2\nreplacement-key\n"), &stderr, false, + ) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, &stderr, prompts) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified || executor.calls != 2 { + t.Fatalf("result/calls = %#v/%d, want auth recovery followed by safe query reconciliation", result, executor.calls) + } + if pippit.key != "replacement-key" || !strings.Contains(stderr.String(), "从安全状态继续") { + t.Fatalf("key/stderr = %q/%q, want safe apply reauthorization", pippit.key, stderr.String()) + } +} + +func TestCanvasImportKeepsQueryingAmbiguousApplyWithoutReplaying(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + executor := &sequencedImportExecutor{ + results: []*canvasplan.ExecutionResult{ + {State: canvasplan.StateApplyAmbiguous, JournalPath: filepath.Join(temp, "ambiguous.json")}, + verifiedImportResult(), + }, + errors: []error{errors.New("temporary exact query-back failure")}, + } + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + deps.mediaPoll = time.Millisecond + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader(""), &stderr, false, + ) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, &stderr, prompts) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified || executor.calls != 2 { + t.Fatalf("result/calls = %#v/%d, want safe read-only recovery", result, executor.calls) + } + if !strings.Contains(stderr.String(), "继续只读回查") { + t.Fatalf("stderr = %q, want read-only recovery progress", stderr.String()) + } +} + +func TestCanvasImportReauthorizesExistingAmbiguousJournal(t *testing.T) { + temp := t.TempDir() + _, exporter, journalPath := prepareSourceBoundResumeFixture(t, temp) + executor := &sequencedReconcileExecutor{ + results: []*canvasplan.ExecutionResult{ + {State: canvasplan.StateApplyAmbiguous, Warning: "query assets failed: ret=1015"}, + verifiedImportResult(), + }, + errors: []error{errors.New("query assets failed: ret=1015")}, + } + pippit := &trackingImportAuthAPI{ + key: "expired-key", + probeErrors: []error{ + nil, + nil, + errors.New("ret=1015"), + nil, + }, + } + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + deps.pippitAuth = pippit + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader("2\nreplacement-key\n"), &stderr, false, + ) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, JournalPath: journalPath, + JournalExplicit: true, AcceptDegradations: true, + }, deps, &stderr, prompts) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified || executor.calls != 2 { + t.Fatalf("result/calls = %#v/%d, want reauthorized journal reconciliation", result, executor.calls) + } + if pippit.key != "replacement-key" || !strings.Contains(stderr.String(), "恢复画布断点期间失效") { + t.Fatalf("key/stderr = %q/%q, want safe journal reauthorization", pippit.key, stderr.String()) + } +} + +func TestCanvasImportKeepsQueryingExistingAmbiguousJournal(t *testing.T) { + temp := t.TempDir() + _, exporter, journalPath := prepareSourceBoundResumeFixture(t, temp) + executor := &sequencedReconcileExecutor{ + results: []*canvasplan.ExecutionResult{ + {State: canvasplan.StateApplyAmbiguous, Warning: "temporary query failure"}, + verifiedImportResult(), + }, + errors: []error{errors.New("temporary query failure")}, + } + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + deps.mediaPoll = time.Millisecond + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader(""), &stderr, false, + ) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, JournalPath: journalPath, + JournalExplicit: true, AcceptDegradations: true, + }, deps, &stderr, prompts) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified || executor.calls != 2 { + t.Fatalf("result/calls = %#v/%d, want safe existing-journal query recovery", result, executor.calls) + } + if !strings.Contains(stderr.String(), "继续只读回查断点") { + t.Fatalf("stderr = %q, want read-only journal recovery progress", stderr.String()) + } +} + +func TestCanvasImportDoesNotRetryLocalReconcileMismatchWithHistoricalWarning(t *testing.T) { + temp := t.TempDir() + _, exporter, journalPath := prepareSourceBoundResumeFixture(t, temp) + executor := &sequencedReconcileExecutor{ + results: []*canvasplan.ExecutionResult{{ + State: canvasplan.StateApplyAmbiguous, + Warning: "historical ambiguous apply warning", + }}, + errors: []error{errors.New("CanvasPlan or resolved media changed after journal creation")}, + } + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, executor) + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader(""), &stderr, false, + ) + + _, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, JournalPath: journalPath, + JournalExplicit: true, AcceptDegradations: true, + }, deps, &stderr, prompts) + if err == nil || !strings.Contains(err.Error(), "changed after journal creation") { + t.Fatalf("runCanvasImport() error = %v, want fail-closed input mismatch", err) + } + if executor.calls != 1 || strings.Contains(stderr.String(), "继续只读回查断点") { + t.Fatalf("calls/stderr = %d/%q, want no retry for local mismatch", executor.calls, stderr.String()) + } +} + +func TestPrepareCanvasImportKeepsPromptSessionForAuthWithExplicitFlags(t *testing.T) { + prepared, prompts, err := prepareCanvasImportOptions( + context.Background(), + strings.NewReader(""), + importOptions{Provider: "libtv", SourceURL: testLibTVURL}, + func(io.Reader) bool { return true }, + io.Discard, + ) + if err != nil { + t.Fatalf("prepareCanvasImportOptions() error = %v", err) + } + if prompts == nil || prepared.SourceURL != testLibTVURL { + t.Fatalf("prepared/prompts = %#v/%v, want auth-capable interactive session", prepared, prompts) + } +} diff --git a/cmd/canvas/import_open_test.go b/cmd/canvas/import_open_test.go new file mode 100644 index 0000000..49e23f9 --- /dev/null +++ b/cmd/canvas/import_open_test.go @@ -0,0 +1,26 @@ +package canvas + +import ( + "reflect" + "testing" +) + +func TestSanitizedCanvasOpenEnvRemovesAccessKeys(t *testing.T) { + input := []string{ + "PATH=/usr/bin", + "HOME=/tmp/home", + "XYQ_ACCESS_KEY=xyq-secret", + "PIPPIT_ACCESS_KEY=pippit-secret", + "PIPPIT_AK=legacy-secret", + "PIPPIT_CLI_PPE_ENV=ppe_cli_canvas_ak", + } + want := []string{ + "PATH=/usr/bin", + "HOME=/tmp/home", + "PIPPIT_CLI_PPE_ENV=ppe_cli_canvas_ak", + } + + if got := sanitizedCanvasOpenEnv(input); !reflect.DeepEqual(got, want) { + t.Fatalf("sanitizedCanvasOpenEnv() = %v, want %v", got, want) + } +} diff --git a/cmd/canvas/import_result_i18n_test.go b/cmd/canvas/import_result_i18n_test.go new file mode 100644 index 0000000..ea3c62f --- /dev/null +++ b/cmd/canvas/import_result_i18n_test.go @@ -0,0 +1,55 @@ +package canvas + +import ( + "testing" + + "github.com/Pippit-dev/pippit-cli/internal/canvasplan" +) + +func TestLocalizeCanvasImportResultRecoveredApply(t *testing.T) { + result := &canvasplan.ExecutionResult{ + State: canvasplan.StateVerified, + Warning: "Canvas apply response was ambiguous", + Verification: &canvasplan.Verification{ + Verified: true, + RecoveredFromQuery: true, + }, + } + + localizeCanvasImportResult(result) + + want := "画布写入响应状态不明确，但已通过精确回读找回并校验全部资产；CLI 未重复提交写入。" + if result.Warning != want { + t.Fatalf("warning = %q, want %q", result.Warning, want) + } +} + +func TestLocalizeCanvasImportResultRecoverableStates(t *testing.T) { + tests := []struct { + name string + state string + want string + }{ + {name: "create pending", state: canvasplan.StateCreatePending, want: "漫剧画布创建请求已受理，服务端仍在处理中；断点已保存，续跑不会重复创建项目。"}, + {name: "create ambiguous", state: canvasplan.StateCreateAmbiguous, want: "漫剧画布创建结果暂不明确；已保存安全断点，请使用同一断点恢复，切勿重新创建。"}, + {name: "apply ambiguous", state: canvasplan.StateApplyAmbiguous, want: "画布写入结果暂不明确；已保存安全断点，恢复时只会回读核对，不会盲目重放写入。"}, + {name: "verification failed", state: canvasplan.StateVerificationFailed, want: "画布写入后的回读验证尚未完成；已保存安全断点，CLI 不会盲目重放写入。"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := &canvasplan.ExecutionResult{State: test.state, Warning: "internal warning"} + localizeCanvasImportResult(result) + if result.Warning != test.want { + t.Fatalf("warning = %q, want %q", result.Warning, test.want) + } + }) + } +} + +func TestLocalizeCanvasImportResultLeavesEmptyWarningAlone(t *testing.T) { + result := &canvasplan.ExecutionResult{State: canvasplan.StateVerified} + localizeCanvasImportResult(result) + if result.Warning != "" { + t.Fatalf("warning = %q, want empty", result.Warning) + } +} diff --git a/cmd/canvas/import_test.go b/cmd/canvas/import_test.go index 3ab50fe..090e43b 100644 --- a/cmd/canvas/import_test.go +++ b/cmd/canvas/import_test.go @@ -31,6 +31,18 @@ type fakeImportExporter struct { mediaBytes map[string][]byte urls []string bundles []string + authCalls int + authErrors []error +} + +func (exporter *fakeImportExporter) Authenticate(context.Context, bool, io.Writer) error { + exporter.authCalls++ + if len(exporter.authErrors) == 0 { + return nil + } + err := exporter.authErrors[0] + exporter.authErrors = exporter.authErrors[1:] + return err } func (exporter *fakeImportExporter) Export( @@ -307,15 +319,15 @@ func TestImportCommandInteractiveWizardUsesSafeDefaults(t *testing.T) { t.Fatalf("opened = %q, want wizard default Yes", opened) } for _, message := range []string{ - "Source provider:", "1) LibTV (default)", "LibTV canvas URL", "Resume journal:", - "1) Automatic (recommended, default)", "2) Custom path", "After import:", - "1) Open Canvas (default)", "2) Do not open", - "Resume journal: " + executor.opts.JournalPath, - `Media progress: processed=1/2 remaining=1 action=uploaded file="one.png"`, - `Media progress: processed=2/2 remaining=0 action=reused file="two.png"`, - `Media progress: processed=0/2 remaining=2 action=uploading file="one.png"`, - "Phase canvas: create/resume, materialize, apply, then verify remote Canvas assets.", - "Phase canvas: Canvas import verified by query-back.", + "导入来源：", "1) LibTV（默认）", "LibTV 画布链接", "断点续跑记录：", + "1) 自动生成（推荐，默认）", "2) 自定义路径", "导入完成后：", + "1) 打开画布（默认）", "2) 暂不打开", + "断点续跑记录：" + executor.opts.JournalPath, + `素材进度：已处理=1/2，剩余=1，状态=已上传，文件="one.png"`, + `素材进度：已处理=2/2，剩余=0，状态=已复用，文件="two.png"`, + `素材进度：已处理=0/2，剩余=2，状态=正在上传，文件="one.png"`, + "阶段：正在创建或续跑画布、写入节点与连线，并回读验证远端画布素材…", + "阶段：画布导入已通过回读验证。", } { if !strings.Contains(stderr.String(), message) { t.Fatalf("stderr missing %q:\n%s", message, stderr.String()) @@ -326,6 +338,26 @@ func TestImportCommandInteractiveWizardUsesSafeDefaults(t *testing.T) { } } +func TestImportCommandHelpUsesChineseCopy(t *testing.T) { + var stdout bytes.Buffer + cmd := newImportCommand(&stdout, io.Discard, importDependencies{}) + cmd.SetArgs([]string{"--help"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + for _, expected := range []string{ + "将外部项目导入个人漫剧画布", + "不传来源参数时会进入交互式向导", + "导入来源（当前仅支持 libtv）", + "来源项目链接", + "断点续跑记录路径（省略时自动生成）", + } { + if !strings.Contains(stdout.String(), expected) { + t.Fatalf("help output missing %q:\n%s", expected, stdout.String()) + } + } +} + func TestImportCommandInteractiveWizardRetriesAndUsesNumberedCustomChoices(t *testing.T) { temp := t.TempDir() journalDirectory := filepath.Join(temp, "custom-state") @@ -355,7 +387,7 @@ func TestImportCommandInteractiveWizardRetriesAndUsesNumberedCustomChoices(t *te if opened { t.Fatal("wizard option 2 unexpectedly opened the Canvas") } - if got := strings.Count(stderr.String(), "Please select a number from 1 to"); got != 3 { + if got := strings.Count(stderr.String(), "请输入 1 到"); got != 3 { t.Fatalf("invalid choice messages = %d, want 3:\n%s", got, stderr.String()) } if !json.Valid(bytes.TrimSpace(stdout.Bytes())) { @@ -377,8 +409,8 @@ func TestImportCommandInteractiveWizardWarnsAndContinuesDegradations(t *testing. if err := cmd.Execute(); err != nil { t.Fatalf("Execute() error = %v, stderr = %s", err, stderr.String()) } - if !strings.Contains(stderr.String(), "known nonfatal degradation(s)") || - !strings.Contains(stderr.String(), "empty-media placeholders or semantic downgrades") || + if !strings.Contains(stderr.String(), "已知的非致命能力降级") || + !strings.Contains(stderr.String(), "空素材占位或语义降级") || !strings.Contains(stderr.String(), "degradation_count") { t.Fatalf("stderr = %q, want auditable automatic degradation warning", stderr.String()) } @@ -406,7 +438,7 @@ func TestImportCommandInteractiveWizardHonorsExplicitDegradationRejection(t *tes if executor.calls != 0 { t.Fatalf("executor calls = %d, want no Canvas write after explicit rejection", executor.calls) } - if strings.Contains(stderr.String(), "continuing the interactive import") { + if strings.Contains(stderr.String(), "交互式导入将自动继续") { t.Fatalf("stderr = %q, explicit false must not be ignored by the wizard", stderr.String()) } } @@ -421,8 +453,8 @@ func TestImportCommandInteractiveCustomJournalEOFIsActionable(t *testing.T) { cmd.SetIn(strings.NewReader("\n" + testLibTVURL + "\n2\n")) cmd.SilenceUsage = true err := cmd.Execute() - if err == nil || !strings.Contains(err.Error(), "custom journal path") || - !strings.Contains(err.Error(), "choose 1 for Automatic") { + if err == nil || !strings.Contains(err.Error(), "自定义断点记录路径") || + !strings.Contains(err.Error(), "请选择 1 自动生成") { t.Fatalf("Execute() error = %v, want actionable custom path EOF", err) } if len(exporter.urls) != 0 { @@ -439,7 +471,7 @@ func TestImportCommandMissingFlagsFailsActionablyWithoutInteractiveInput(t *test cmd.SetIn(strings.NewReader("")) cmd.SilenceUsage = true err := cmd.Execute() - if err == nil || !strings.Contains(err.Error(), "stdin is not interactive") || + if err == nil || !strings.Contains(err.Error(), "当前输入不是交互式终端") || !strings.Contains(err.Error(), "--from libtv --url") { t.Fatalf("Execute() error = %v, want actionable non-interactive flags", err) } @@ -469,7 +501,7 @@ func TestImportCommandInteractiveEOFMissingURLFailsBeforeExport(t *testing.T) { cmd.SetIn(strings.NewReader("\n")) cmd.SilenceUsage = true err := cmd.Execute() - if err == nil || !strings.Contains(err.Error(), "ended before a LibTV URL") || + if err == nil || !strings.Contains(err.Error(), "尚未提供 LibTV 画布链接") || !strings.Contains(err.Error(), "--from libtv --url") { t.Fatalf("Execute() error = %v, want actionable EOF guidance", err) } @@ -562,9 +594,9 @@ func TestImportCommandWaitsForProcessingUploadAndContinuesSameInvocation(t *test t.Fatalf("upload/query/execute = %d/%d/%d, want 1/3/1 in one invocation", media.uploads, media.queries, executor.calls) } for _, progress := range []string{ - `Media progress: processed=0/1 remaining=1 action=processing file="one.png"`, - `Media progress: processed=0/1 remaining=1 action=waiting file="one.png"`, - `Media progress: processed=1/1 remaining=0 action=uploaded file="one.png"`, + `素材进度：已处理=0/1，剩余=1，状态=正在处理，文件="one.png"`, + `素材进度：已处理=0/1，剩余=1，状态=等待处理中，文件="one.png"`, + `素材进度：已处理=1/1，剩余=0，状态=已上传，文件="one.png"`, } { if !strings.Contains(stderr.String(), progress) { t.Fatalf("stderr = %q, want progress %q", stderr.String(), progress) @@ -575,6 +607,37 @@ func TestImportCommandWaitsForProcessingUploadAndContinuesSameInvocation(t *test } } +func TestImportCommandContinuesAfterInteractiveMediaProcessingWindow(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testSingleMediaPlan(t) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &fakeImportMediaAPI{ + uploadState: canvascore.StateProcessing, + queryReady: []bool{false, true}, + } + executor := &fakeImportExecutor{result: verifiedImportResult()} + deps := testImportDependencies(temp, exporter, media, executor) + deps.mediaPoll = 2 * time.Millisecond + deps.mediaTimeout = time.Millisecond + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI( + context.Background(), strings.NewReader(""), &stderr, false, + ) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, &stderr, prompts) + if err != nil { + t.Fatalf("runCanvasImport() error = %v, stderr = %s", err, stderr.String()) + } + if result == nil || result.State != canvasplan.StateVerified || media.uploads != 1 || media.queries != 2 { + t.Fatalf("result/upload/query = %#v/%d/%d, want one upload and continued queries", result, media.uploads, media.queries) + } + if !strings.Contains(stderr.String(), "继续只读查询，不会重复上传") { + t.Fatalf("stderr = %q, want noninterrupting processing progress", stderr.String()) + } +} + func TestImportMediaResumesProcessingCheckpointWithoutUploading(t *testing.T) { opts := testMediaResolutionOptions(t) item := opts.Media[0] @@ -624,8 +687,8 @@ func TestImportMediaProcessingQueryAuthErrorStopsAndPreservesIDs(t *testing.T) { } api := &fakeImportMediaAPI{queryErr: errors.New("HTTP 401 Unauthorized")} _, err := resolveImportMedia(context.Background(), opts, api, io.Discard) - if err == nil || !strings.Contains(err.Error(), "read/authentication error") || - !strings.Contains(err.Error(), "401 Unauthorized") || !strings.Contains(err.Error(), "durable IDs remain") { + if err == nil || !strings.Contains(err.Error(), "读取或授权错误") || + !strings.Contains(err.Error(), "401 Unauthorized") || !strings.Contains(err.Error(), "持久化素材 ID 仍保存在") { t.Fatalf("resolveImportMedia() error = %v, want immediate explicit auth/query error", err) } if api.uploads != 0 || api.queries != 1 { @@ -774,7 +837,7 @@ func TestImportMediaRejectsLegacyPNGCheckpointWhenPixelsChange(t *testing.T) { opts, oldSHA := testPNGCheckpointMigrationOptions(t, oldPNG, currentPNG) api := &fakeImportMediaAPI{} _, err := resolveImportMedia(context.Background(), opts, api, io.Discard) - if err == nil || !strings.Contains(err.Error(), "image content changed") { + if err == nil || !strings.Contains(err.Error(), "图片内容在断点记录创建后发生了变化") { t.Fatalf("resolveImportMedia() error = %v, want normalized-content mismatch rejection", err) } if api.uploads != 0 || api.queries != 0 { @@ -815,7 +878,7 @@ func TestImportMediaProgressReportsEmptySet(t *testing.T) { } if len(resolved.Media) != 0 || !strings.Contains( stderr.String(), - `Media progress: processed=0/0 remaining=0 action=complete file="(none)"`, + `素材进度：已处理=0/0，剩余=0，状态=完成，文件="（无）"`, ) { t.Fatalf("resolved/stderr = %#v/%q, want explicit 0/0 progress", resolved, stderr.String()) } @@ -878,7 +941,7 @@ func TestMediaCheckpointDoesNotMarkMissingAKAsUploadRequested(t *testing.T) { opts := testMediaResolutionOptions(t) api := &missingAKPreflightMediaAPI{} _, err := resolveImportMedia(context.Background(), opts, api, io.Discard) - if err == nil || !strings.Contains(err.Error(), "authentication failed") { + if err == nil || !strings.Contains(err.Error(), "授权检查失败") { t.Fatalf("resolveImportMedia() error = %v, want explicit authentication failure", err) } if api.uploads != 0 { @@ -935,7 +998,7 @@ func TestReadyMediaCheckpointMustBelongToCurrentPippitAccount(t *testing.T) { } api := &fakeImportMediaAPI{queryErr: errors.New("asset not found for current account")} _, err := resolveImportMedia(context.Background(), opts, api, io.Discard) - if err == nil || !strings.Contains(err.Error(), "current Pippit account") { + if err == nil || !strings.Contains(err.Error(), "当前小云雀账号") { t.Fatalf("resolveImportMedia() error = %v, want cross-account checkpoint rejection", err) } if api.queries != 1 || api.uploads != 0 { @@ -1053,6 +1116,8 @@ func testImportDependencies( executor importExecutor, ) importDependencies { return importDependencies{ + pippitAuth: &fakeImportAuthAPI{accessKey: "test-access-key"}, + sourceAuth: exporter.(importSourceAuthenticator), exporter: exporter, media: media, executor: executor, From 998f51075119f952f3eea26d5fab6685dcdcbf01 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 07:08:58 +0800 Subject: [PATCH 36/48] feat(auth): add secure CLI credential storage Co-authored-by: Codex <codex@openai.com> --- go.mod | 6 +- go.sum | 7 + internal/auth/browser_darwin.go | 14 ++ internal/auth/browser_env.go | 35 ++++ internal/auth/browser_linux.go | 14 ++ internal/auth/browser_windows.go | 14 ++ internal/auth/identity.go | 92 +++++++++ internal/auth/store.go | 300 ++++++++++++++++++++++++++++ internal/auth/store_file_unix.go | 176 ++++++++++++++++ internal/auth/store_file_windows.go | 11 + internal/auth/types.go | 77 +++++++ 11 files changed, 744 insertions(+), 2 deletions(-) create mode 100644 internal/auth/browser_darwin.go create mode 100644 internal/auth/browser_env.go create mode 100644 internal/auth/browser_linux.go create mode 100644 internal/auth/browser_windows.go create mode 100644 internal/auth/identity.go create mode 100644 internal/auth/store.go create mode 100644 internal/auth/store_file_unix.go create mode 100644 internal/auth/store_file_windows.go create mode 100644 internal/auth/types.go diff --git a/go.mod b/go.mod index 26c14fc..d320fba 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.23.0 require ( github.com/bytedance/sonic v1.15.1 github.com/charmbracelet/huh v1.0.0 - github.com/charmbracelet/x/term v0.2.1 github.com/spf13/cobra v1.8.1 + github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.33.0 ) @@ -23,9 +23,12 @@ require ( github.com/charmbracelet/x/ansi v0.9.3 // indirect github.com/charmbracelet/x/cellbuf v0.0.13 // indirect github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect github.com/cloudwego/base64x v0.1.6 // indirect + github.com/danieljoos/wincred v1.2.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect @@ -38,7 +41,6 @@ require ( github.com/muesli/termenv v0.16.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect diff --git a/go.sum b/go.sum index e145c77..3587564 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,8 @@ github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gE github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= +github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -54,6 +56,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= +github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= @@ -87,6 +91,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -98,6 +103,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= +github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= diff --git a/internal/auth/browser_darwin.go b/internal/auth/browser_darwin.go new file mode 100644 index 0000000..a64fab1 --- /dev/null +++ b/internal/auth/browser_darwin.go @@ -0,0 +1,14 @@ +//go:build darwin + +package auth + +import ( + "os" + "os/exec" +) + +func OpenBrowser(rawURL string) error { + command := exec.Command("open", rawURL) + command.Env = SanitizedBrowserEnv(os.Environ()) + return command.Start() +} diff --git a/internal/auth/browser_env.go b/internal/auth/browser_env.go new file mode 100644 index 0000000..dde9c41 --- /dev/null +++ b/internal/auth/browser_env.go @@ -0,0 +1,35 @@ +package auth + +import "strings" + +// SanitizedBrowserEnv removes Pippit/XYQ credential variables before spawning +// a browser helper. The login URL carries its own one-time binding and does not +// need any CLI credential from the child process environment. +func SanitizedBrowserEnv(environment []string) []string { + result := make([]string, 0, len(environment)) + for _, entry := range environment { + name, _, found := strings.Cut(entry, "=") + if !found || isCredentialEnvName(name) { + continue + } + result = append(result, entry) + } + return result +} + +func isCredentialEnvName(name string) bool { + upper := strings.ToUpper(strings.TrimSpace(name)) + if !strings.HasPrefix(upper, "PIPPIT_") && !strings.HasPrefix(upper, "XYQ_") { + return false + } + if strings.Contains(upper, "ACCESS_KEY") || strings.Contains(upper, "TOKEN") || strings.Contains(upper, "SECRET") { + return true + } + parts := strings.Split(upper, "_") + for _, part := range parts { + if part == "AK" { + return true + } + } + return false +} diff --git a/internal/auth/browser_linux.go b/internal/auth/browser_linux.go new file mode 100644 index 0000000..f147b2c --- /dev/null +++ b/internal/auth/browser_linux.go @@ -0,0 +1,14 @@ +//go:build linux + +package auth + +import ( + "os" + "os/exec" +) + +func OpenBrowser(rawURL string) error { + command := exec.Command("xdg-open", rawURL) + command.Env = SanitizedBrowserEnv(os.Environ()) + return command.Start() +} diff --git a/internal/auth/browser_windows.go b/internal/auth/browser_windows.go new file mode 100644 index 0000000..c685d88 --- /dev/null +++ b/internal/auth/browser_windows.go @@ -0,0 +1,14 @@ +//go:build windows + +package auth + +import ( + "os" + "os/exec" +) + +func OpenBrowser(rawURL string) error { + command := exec.Command("rundll32", "url.dll,FileProtocolHandler", rawURL) + command.Env = SanitizedBrowserEnv(os.Environ()) + return command.Start() +} diff --git a/internal/auth/identity.go b/internal/auth/identity.go new file mode 100644 index 0000000..4882800 --- /dev/null +++ b/internal/auth/identity.go @@ -0,0 +1,92 @@ +package auth + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "io" + "strings" +) + +func randomEncoded(reader io.Reader, size int) (string, error) { + if reader == nil { + reader = rand.Reader + } + value := make([]byte, size) + if _, err := io.ReadFull(reader, value); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(value), nil +} + +func randomTempName() (string, error) { + value, err := randomEncoded(rand.Reader, 18) + if err != nil { + return "", err + } + return ".credential-" + value + ".tmp", nil +} + +func validDeviceID(deviceID string) bool { + decoded, err := base64.RawURLEncoding.DecodeString(deviceID) + return err == nil && len(decoded) == deviceIDBytes && + constantTimeEqual(deviceID, base64.RawURLEncoding.EncodeToString(decoded)) +} + +func tokenNameForDevice(deviceID string) string { + digest := sha256.Sum256([]byte(deviceID)) + return "pippit-tool-cli-" + base64.RawURLEncoding.EncodeToString(digest[:16]) +} + +func credentialScope(uid, deviceID string) string { + digest := sha256.Sum256([]byte(strings.TrimSpace(uid))) + return "pippit-tool-cli:user:" + base64.RawURLEncoding.EncodeToString(digest[:16]) + ":device:" + deviceID +} + +func legacyCredentialScope(deviceID string) string { + return "pippit-tool-cli:device:" + deviceID +} + +func constantTimeEqual(left, right string) bool { + leftHash := sha256.Sum256([]byte(left)) + rightHash := sha256.Sum256([]byte(right)) + return subtle.ConstantTimeCompare(leftHash[:], rightHash[:]) == 1 +} + +func newIdentity(reader io.Reader) (*Credential, error) { + deviceID, err := randomEncoded(reader, deviceIDBytes) + if err != nil { + return nil, errors.New("生成本机登录设备标识失败") + } + return &Credential{ + Version: credentialVersion, + DeviceID: deviceID, + TokenName: tokenNameForDevice(deviceID), + }, nil +} + +func identityOnly(credential *Credential) *Credential { + if credential == nil { + return nil + } + return &Credential{ + Version: credential.Version, + DeviceID: credential.DeviceID, + TokenName: credential.TokenName, + // TokenID is not an authentication secret. Keeping this exact remote + // reference lets a later login reuse or rotate the same device token + // without consuming another per-account AK slot. + TokenID: credential.TokenID, + } +} + +func redactedOperationError(operation string) error { + operation = strings.TrimSpace(operation) + if operation == "" { + operation = "授权操作" + } + return fmt.Errorf("%s失败，请稍后重试", operation) +} diff --git a/internal/auth/store.go b/internal/auth/store.go new file mode 100644 index 0000000..970310b --- /dev/null +++ b/internal/auth/store.go @@ -0,0 +1,300 @@ +package auth + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/zalando/go-keyring" +) + +const ( + defaultKeyringAccount = "browser-login" + credentialFileName = "browser-credential.json" + maxCredentialBytes = 64 << 10 +) + +type keyringBackend interface { + Get(service, user string) (string, error) + Set(service, user, password string) error + Delete(service, user string) error +} + +type systemKeyring struct{} + +func (systemKeyring) Get(service, user string) (string, error) { + return keyring.Get(service, user) +} + +func (systemKeyring) Set(service, user, password string) error { + return keyring.Set(service, user, password) +} + +func (systemKeyring) Delete(service, user string) error { + return keyring.Delete(service, user) +} + +type keyringCredentialStore struct { + backend keyringBackend + service string + account string +} + +func (s *keyringCredentialStore) Load(ctx context.Context) (*Credential, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + value, err := s.backend.Get(s.service, s.account) + if errors.Is(err, keyring.ErrNotFound) { + return nil, ErrCredentialNotFound + } + if err != nil { + return nil, fmt.Errorf("读取系统钥匙串失败: %w", ErrSecureStore) + } + credential, err := decodeCredential([]byte(value)) + if err != nil { + return nil, err + } + return credential, nil +} + +func (s *keyringCredentialStore) Save(ctx context.Context, credential *Credential) error { + if err := ctx.Err(); err != nil { + return err + } + payload, err := encodeCredential(credential) + if err != nil { + return err + } + if err := s.backend.Set(s.service, s.account, string(payload)); err != nil { + return fmt.Errorf("写入系统钥匙串失败: %w", ErrSecureStore) + } + return nil +} + +func (s *keyringCredentialStore) Delete(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + err := s.backend.Delete(s.service, s.account) + if errors.Is(err, keyring.ErrNotFound) { + return ErrCredentialNotFound + } + if err != nil { + return fmt.Errorf("删除系统钥匙串凭证失败: %w", ErrSecureStore) + } + return nil +} + +type resilientCredentialStore struct { + primary CredentialStore + fallback CredentialStore +} + +type storedCredential struct { + Version int `json:"version"` + DeviceID string `json:"device_id"` + CredentialScope string `json:"credential_scope"` + TokenName string `json:"token_name"` + AccessKey string `json:"access_key,omitempty"` + TokenID string `json:"token_id,omitempty"` + UID string `json:"uid,omitempty"` + ExpiredAt int64 `json:"expired_at,omitempty"` +} + +// NewDefaultCredentialStore prefers the operating system keyring. Unix uses +// a private no-follow file as a fallback; Windows deliberately has no file +// fallback because an equivalent ACL guarantee is not provided here. +func NewDefaultCredentialStore(serviceName string) CredentialStore { + serviceName = strings.TrimSpace(serviceName) + if serviceName == "" { + serviceName = "pippit-cli" + } + primary := &keyringCredentialStore{ + backend: systemKeyring{}, + service: serviceName, + account: defaultKeyringAccount, + } + return &resilientCredentialStore{ + primary: primary, + fallback: newPlatformFallbackStore(), + } +} + +func (s *resilientCredentialStore) Load(ctx context.Context) (*Credential, error) { + credential, primaryErr := s.primary.Load(ctx) + if primaryErr == nil { + return credential, nil + } + if err := ctx.Err(); err != nil { + return nil, err + } + if !errors.Is(primaryErr, ErrCredentialNotFound) && !errors.Is(primaryErr, ErrSecureStore) { + // A decodable-but-invalid primary record may indicate corruption or + // tampering. Never mask it with an older fallback credential. + return nil, primaryErr + } + if s.fallback == nil { + if errors.Is(primaryErr, ErrCredentialNotFound) { + return nil, ErrCredentialNotFound + } + return nil, primaryErr + } + credential, fallbackErr := s.fallback.Load(ctx) + if fallbackErr == nil { + return credential, nil + } + if errors.Is(primaryErr, ErrCredentialNotFound) && errors.Is(fallbackErr, ErrCredentialNotFound) { + return nil, ErrCredentialNotFound + } + if !errors.Is(fallbackErr, ErrCredentialNotFound) { + return nil, fallbackErr + } + return nil, primaryErr +} + +func (s *resilientCredentialStore) Save(ctx context.Context, credential *Credential) error { + primaryErr := s.primary.Save(ctx, credential) + if primaryErr == nil { + if s.fallback != nil { + if err := ignoreNotFound(s.fallback.Delete(ctx)); err != nil { + return fmt.Errorf("系统钥匙串已更新，但清理旧的备用凭证失败: %w", err) + } + } + return nil + } + if err := ctx.Err(); err != nil { + return err + } + if !errors.Is(primaryErr, ErrCredentialNotFound) && !errors.Is(primaryErr, ErrSecureStore) { + return primaryErr + } + if s.fallback == nil { + return primaryErr + } + if err := s.fallback.Save(ctx, credential); err != nil { + return err + } + return nil +} + +func (s *resilientCredentialStore) Delete(ctx context.Context) error { + primaryErr := ignoreNotFound(s.primary.Delete(ctx)) + var fallbackErr error + if s.fallback != nil { + fallbackErr = ignoreNotFound(s.fallback.Delete(ctx)) + } + if primaryErr != nil { + return primaryErr + } + return fallbackErr +} + +func ignoreNotFound(err error) error { + if errors.Is(err, ErrCredentialNotFound) { + return nil + } + return err +} + +func encodeCredential(credential *Credential) ([]byte, error) { + if err := validateCredential(credential); err != nil { + return nil, err + } + payload, err := json.Marshal(storedCredential{ + Version: credential.Version, + DeviceID: credential.DeviceID, + CredentialScope: credential.CredentialScope, + TokenName: credential.TokenName, + AccessKey: credential.AccessKey, + TokenID: credential.TokenID, + UID: credential.UID, + ExpiredAt: credential.ExpiredAt, + }) + if err != nil { + return nil, errors.New("编码本机登录凭证失败") + } + return payload, nil +} + +func decodeCredential(payload []byte) (*Credential, error) { + if len(payload) == 0 || len(payload) > maxCredentialBytes { + return nil, errors.New("本机登录凭证格式无效") + } + record := &storedCredential{} + if err := json.Unmarshal(payload, record); err != nil { + return nil, errors.New("本机登录凭证格式无效") + } + credential := &Credential{ + Version: record.Version, + DeviceID: record.DeviceID, + CredentialScope: record.CredentialScope, + TokenName: record.TokenName, + AccessKey: record.AccessKey, + TokenID: record.TokenID, + UID: record.UID, + ExpiredAt: record.ExpiredAt, + } + if err := validateCredential(credential); err != nil { + return nil, err + } + return credential, nil +} + +func defaultCredentialPath() string { + dir, err := os.UserConfigDir() + if err != nil || strings.TrimSpace(dir) == "" { + return "" + } + return filepath.Join(dir, "pippit-cli", credentialFileName) +} + +type unavailableCredentialStore struct{} + +func (unavailableCredentialStore) Load(context.Context) (*Credential, error) { + return nil, ErrSecureStore +} + +func (unavailableCredentialStore) Save(context.Context, *Credential) error { + return ErrSecureStore +} + +func (unavailableCredentialStore) Delete(context.Context) error { + return ErrSecureStore +} + +func validateCredential(credential *Credential) error { + if credential == nil { + return errors.New("本机登录凭证不能为空") + } + if credential.Version != credentialVersion { + return errors.New("本机登录凭证版本不受支持") + } + if !validDeviceID(credential.DeviceID) { + return errors.New("本机登录设备标识无效") + } + if !constantTimeEqual(credential.TokenName, tokenNameForDevice(credential.DeviceID)) { + return errors.New("本机登录凭证作用域无效") + } + if credential.AccessKey == "" { + if strings.TrimSpace(credential.TokenID) != credential.TokenID || credential.UID != "" || credential.ExpiredAt != 0 || + (credential.CredentialScope != "" && !constantTimeEqual(credential.CredentialScope, legacyCredentialScope(credential.DeviceID))) { + return errors.New("本机登录凭证不完整") + } + return nil + } + if strings.TrimSpace(credential.AccessKey) != credential.AccessKey || + credential.TokenID == "" || credential.UID == "" || credential.ExpiredAt <= 0 { + return errors.New("本机登录凭证不完整") + } + expectedScope := credentialScope(credential.UID, credential.DeviceID) + if !constantTimeEqual(credential.CredentialScope, expectedScope) && + !constantTimeEqual(credential.CredentialScope, legacyCredentialScope(credential.DeviceID)) { + return errors.New("本机登录凭证作用域无效") + } + return nil +} diff --git a/internal/auth/store_file_unix.go b/internal/auth/store_file_unix.go new file mode 100644 index 0000000..275621c --- /dev/null +++ b/internal/auth/store_file_unix.go @@ -0,0 +1,176 @@ +//go:build !windows + +package auth + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" +) + +type fileCredentialStore struct { + path string +} + +func NewFileCredentialStore(path string) CredentialStore { + return &fileCredentialStore{path: filepath.Clean(path)} +} + +func newPlatformFallbackStore() CredentialStore { + path := defaultCredentialPath() + if path == "" { + return nil + } + return NewFileCredentialStore(path) +} + +func (s *fileCredentialStore) Load(ctx context.Context) (*Credential, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + dirFD, name, err := s.openPrivateDirectory(false) + if errors.Is(err, os.ErrNotExist) { + return nil, ErrCredentialNotFound + } + if err != nil { + return nil, err + } + defer unix.Close(dirFD) + + fd, err := unix.Openat(dirFD, name, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if errors.Is(err, unix.ENOENT) { + return nil, ErrCredentialNotFound + } + if err != nil { + return nil, fmt.Errorf("安全打开本机登录凭证失败: %w", ErrSecureStore) + } + file := os.NewFile(uintptr(fd), name) + defer file.Close() + if err := verifyPrivateRegularFile(fd); err != nil { + return nil, err + } + payload, err := io.ReadAll(io.LimitReader(file, maxCredentialBytes+1)) + if err != nil { + return nil, fmt.Errorf("读取本机登录凭证失败: %w", ErrSecureStore) + } + return decodeCredential(payload) +} + +func (s *fileCredentialStore) Save(ctx context.Context, credential *Credential) error { + if err := ctx.Err(); err != nil { + return err + } + payload, err := encodeCredential(credential) + if err != nil { + return err + } + dirFD, name, err := s.openPrivateDirectory(true) + if err != nil { + return err + } + defer unix.Close(dirFD) + + tempName, err := randomTempName() + if err != nil { + return errors.New("创建本机登录凭证临时文件失败") + } + fd, err := unix.Openat(dirFD, tempName, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) + if err != nil { + return fmt.Errorf("创建本机登录凭证临时文件失败: %w", ErrSecureStore) + } + cleanup := true + defer func() { + if cleanup { + _ = unix.Unlinkat(dirFD, tempName, 0) + } + }() + file := os.NewFile(uintptr(fd), tempName) + if _, err := file.Write(payload); err != nil { + _ = file.Close() + return fmt.Errorf("写入本机登录凭证失败: %w", ErrSecureStore) + } + if err := file.Sync(); err != nil { + _ = file.Close() + return fmt.Errorf("同步本机登录凭证失败: %w", ErrSecureStore) + } + if err := file.Close(); err != nil { + return fmt.Errorf("关闭本机登录凭证失败: %w", ErrSecureStore) + } + if err := unix.Renameat(dirFD, tempName, dirFD, name); err != nil { + return fmt.Errorf("原子保存本机登录凭证失败: %w", ErrSecureStore) + } + cleanup = false + if err := unix.Fsync(dirFD); err != nil { + return fmt.Errorf("同步本机凭证目录失败: %w", ErrSecureStore) + } + return nil +} + +func (s *fileCredentialStore) Delete(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + dirFD, name, err := s.openPrivateDirectory(false) + if errors.Is(err, os.ErrNotExist) { + return ErrCredentialNotFound + } + if err != nil { + return err + } + defer unix.Close(dirFD) + if err := unix.Unlinkat(dirFD, name, 0); errors.Is(err, unix.ENOENT) { + return ErrCredentialNotFound + } else if err != nil { + return fmt.Errorf("删除本机登录凭证失败: %w", ErrSecureStore) + } + if err := unix.Fsync(dirFD); err != nil { + return fmt.Errorf("同步本机凭证目录失败: %w", ErrSecureStore) + } + return nil +} + +func (s *fileCredentialStore) openPrivateDirectory(create bool) (int, string, error) { + if s == nil || strings.TrimSpace(s.path) == "" || filepath.Base(s.path) == "." { + return -1, "", ErrSecureStore + } + dir := filepath.Dir(s.path) + if create { + if err := os.MkdirAll(dir, 0o700); err != nil { + return -1, "", fmt.Errorf("创建本机凭证目录失败: %w", ErrSecureStore) + } + } + info, err := os.Lstat(dir) + if err != nil { + return -1, "", err + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() || info.Mode().Perm()&0o077 != 0 { + return -1, "", fmt.Errorf("本机凭证目录必须是权限 0700 的真实目录: %w", ErrSecureStore) + } + dirFD, err := unix.Open(dir, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_DIRECTORY|unix.O_NOFOLLOW, 0) + if err != nil { + return -1, "", fmt.Errorf("安全打开本机凭证目录失败: %w", ErrSecureStore) + } + var stat unix.Stat_t + if err := unix.Fstat(dirFD, &stat); err != nil || stat.Uid != uint32(os.Geteuid()) { + unix.Close(dirFD) + return -1, "", fmt.Errorf("本机凭证目录所有者无效: %w", ErrSecureStore) + } + return dirFD, filepath.Base(s.path), nil +} + +func verifyPrivateRegularFile(fd int) error { + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + return fmt.Errorf("检查本机登录凭证失败: %w", ErrSecureStore) + } + if stat.Uid != uint32(os.Geteuid()) || stat.Mode&unix.S_IFMT != unix.S_IFREG || stat.Mode&0o077 != 0 { + return fmt.Errorf("本机登录凭证必须是当前用户拥有的权限 0600 文件: %w", ErrSecureStore) + } + return nil +} diff --git a/internal/auth/store_file_windows.go b/internal/auth/store_file_windows.go new file mode 100644 index 0000000..84143bf --- /dev/null +++ b/internal/auth/store_file_windows.go @@ -0,0 +1,11 @@ +//go:build windows + +package auth + +func NewFileCredentialStore(string) CredentialStore { + return unavailableCredentialStore{} +} + +func newPlatformFallbackStore() CredentialStore { + return nil +} diff --git a/internal/auth/types.go b/internal/auth/types.go new file mode 100644 index 0000000..8786e3f --- /dev/null +++ b/internal/auth/types.go @@ -0,0 +1,77 @@ +package auth + +import ( + "context" + "errors" + "io" + "time" +) + +const ( + loginExportPath = "/cli/login-export" + callbackPath = "/xyq/callback/save_session" + exchangeGrantPath = "/api/web/v1/auth/exchange_cli_login_grant" + queryAccessKeyPath = "/api/biz/v1/user/query_ak" + generateAccessKeyPath = "/api/biz/v1/user/generate_ak" + deleteAccessKeyPath = "/api/biz/v1/user/delete_ak" + loginSource = "pippit-tool-cli" + credentialVersion = 1 + deviceIDBytes = 32 + randomBindingBytes = 32 + + DefaultLoginTimeout = 5 * time.Minute + DefaultCredentialLifetime = 365 * 24 * time.Hour +) + +var ( + ErrCredentialNotFound = errors.New("未找到本机小云雀 CLI 登录凭证") + ErrCredentialExpired = errors.New("本机小云雀 CLI 登录凭证已过期") + ErrSecureStore = errors.New("安全凭证存储不可用") + ErrCredentialAccountMismatch = errors.New("网页授权账号与当前任务账号不一致") + ErrRemoteRevokeUnsupported = errors.New("当前版本不支持在 CLI 中安全撤销远程 Access Key") +) + +// Credential is the dedicated, device-scoped Access Key managed by this CLI. +// AccessKey is secret and must never be printed, logged, or written to journals. +type Credential struct { + Version int `json:"version"` + DeviceID string `json:"device_id"` + CredentialScope string `json:"credential_scope"` + TokenName string `json:"token_name"` + AccessKey string `json:"-"` + TokenID string `json:"token_id,omitempty"` + UID string `json:"uid,omitempty"` + ExpiredAt int64 `json:"expired_at,omitempty"` +} + +// CredentialStore persists a credential without exposing its serialized form. +type CredentialStore interface { + Load(context.Context) (*Credential, error) + Save(context.Context, *Credential) error + Delete(context.Context) error +} + +type LoginOptions struct { + // OpenURL should open the URL without logging it. When nil, the platform's + // standard browser opener is used with credential-bearing env vars removed. + OpenURL func(string) error + Progress io.Writer + Timeout time.Duration + // ForceRefresh rotates every remote token owned by this device identity + // before provisioning a replacement. It is used after an explicit 401 so + // a successful browser login can never return the just-rejected AK again. + ForceRefresh bool + // ExpectedCredentialScope binds reauthentication to the UID and device that + // started a durable operation. A different browser account fails before any + // Access Key is deleted or generated. + ExpectedCredentialScope string +} + +type Status struct { + LoggedIn bool `json:"logged_in"` + Source string `json:"source,omitempty"` + UID string `json:"uid,omitempty"` + TokenID string `json:"token_id,omitempty"` + CredentialScope string `json:"credential_scope,omitempty"` + ExpiresAt time.Time `json:"expires_at,omitempty"` +} From 47886cbfe17439653d0348a2545736c2b8dee95f Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 07:11:05 +0800 Subject: [PATCH 37/48] feat(auth): harden the browser callback flow Co-authored-by: Codex <codex@openai.com> --- internal/auth/loopback.go | 243 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 internal/auth/loopback.go diff --git a/internal/auth/loopback.go b/internal/auth/loopback.go new file mode 100644 index 0000000..ac34136 --- /dev/null +++ b/internal/auth/loopback.go @@ -0,0 +1,243 @@ +package auth + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +const maxCallbackBodyBytes = 64 << 10 + +type loginGrantPayload struct { + Type string `json:"type"` + Grant string `json:"grant"` + RandomSecretKey string `json:"random_secret_key"` + ExpireAt int64 `json:"expire_at,omitempty"` + Source string `json:"source"` + CallbackURL string `json:"callback_url"` +} + +type browserFlow struct { + loginURL string + callbackURL string + secret string + state string + source string + origin string + listener net.Listener + server *http.Server + payload chan loginGrantPayload + serveErr chan error + closeOnce sync.Once +} + +func startBrowserFlow(authBaseURL *url.URL, randomReader io.Reader) (*browserFlow, error) { + secret, err := randomEncoded(randomReader, randomBindingBytes) + if err != nil { + return nil, errors.New("生成网页授权绑定信息失败") + } + state, err := randomEncoded(randomReader, randomBindingBytes) + if err != nil { + return nil, errors.New("生成网页授权状态失败") + } + listener, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + return nil, errors.New("启动本机网页授权回调失败") + } + + callback := &url.URL{ + Scheme: "http", + Host: listener.Addr().String(), + Path: callbackPath, + } + callbackQuery := callback.Query() + callbackQuery.Set("state", state) + callback.RawQuery = callbackQuery.Encode() + + loginURL := *authBaseURL + loginURL.Path = loginExportPath + loginURL.RawPath = "" + loginURL.RawQuery = "" + loginURL.Fragment = "" + query := loginURL.Query() + query.Set("callback", callback.String()) + query.Set("random_secret_key", secret) + query.Set("source", loginSource) + loginURL.RawQuery = query.Encode() + + flow := &browserFlow{ + loginURL: loginURL.String(), + callbackURL: callback.String(), + secret: secret, + state: state, + source: loginSource, + origin: originOf(authBaseURL), + listener: listener, + payload: make(chan loginGrantPayload, 1), + serveErr: make(chan error, 1), + } + mux := http.NewServeMux() + mux.HandleFunc(callbackPath, flow.handleCallback) + flow.server = &http.Server{ + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 15 * time.Second, + } + go func() { + err := flow.server.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + flow.serveErr <- err + } + close(flow.serveErr) + }() + return flow, nil +} + +func (f *browserFlow) wait(ctx context.Context) (loginGrantPayload, error) { + select { + case payload := <-f.payload: + return payload, nil + case err, open := <-f.serveErr: + if open && err != nil { + return loginGrantPayload{}, errors.New("本机网页授权回调异常退出") + } + return loginGrantPayload{}, errors.New("本机网页授权回调已关闭") + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return loginGrantPayload{}, errors.New("等待网页授权超时，请重新登录") + } + return loginGrantPayload{}, ctx.Err() + } +} + +func (f *browserFlow) close() { + f.closeOnce.Do(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = f.server.Shutdown(ctx) + _ = f.listener.Close() + }) +} + +func (f *browserFlow) handleCallback(writer http.ResponseWriter, request *http.Request) { + if !f.validRequestTarget(request) { + http.Error(writer, "invalid callback target", http.StatusBadRequest) + return + } + if !constantTimeEqual(request.Header.Get("Origin"), f.origin) { + http.Error(writer, "origin not allowed", http.StatusForbidden) + return + } + f.setCORSHeaders(writer.Header()) + + if request.Method == http.MethodOptions { + if !strings.EqualFold(strings.TrimSpace(request.Header.Get("Access-Control-Request-Method")), http.MethodPost) || + !allowsContentTypeHeader(request.Header.Get("Access-Control-Request-Headers")) { + http.Error(writer, "invalid preflight", http.StatusBadRequest) + return + } + writer.WriteHeader(http.StatusNoContent) + return + } + if request.Method != http.MethodPost { + writer.Header().Set("Allow", "OPTIONS, POST") + http.Error(writer, "method not allowed", http.StatusMethodNotAllowed) + return + } + mediaType, _, err := mime.ParseMediaType(request.Header.Get("Content-Type")) + if err != nil || !strings.EqualFold(mediaType, "application/json") { + http.Error(writer, "content type must be application/json", http.StatusUnsupportedMediaType) + return + } + + reader := http.MaxBytesReader(writer, request.Body, maxCallbackBodyBytes) + decoder := json.NewDecoder(reader) + decoder.DisallowUnknownFields() + payload := loginGrantPayload{} + if err := decoder.Decode(&payload); err != nil { + http.Error(writer, "invalid callback payload", http.StatusBadRequest) + return + } + if err := ensureJSONEOF(decoder); err != nil { + http.Error(writer, "invalid callback payload", http.StatusBadRequest) + return + } + if payload.Type != "login_grant" || strings.TrimSpace(payload.Grant) == "" || + !constantTimeEqual(payload.RandomSecretKey, f.secret) || + !constantTimeEqual(payload.Source, f.source) || + !constantTimeEqual(payload.CallbackURL, f.callbackURL) { + http.Error(writer, "callback binding mismatch", http.StatusBadRequest) + return + } + select { + case f.payload <- payload: + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusOK) + _, _ = writer.Write([]byte(`{"ok":true}`)) + default: + http.Error(writer, "callback already received", http.StatusConflict) + } +} + +func (f *browserFlow) validRequestTarget(request *http.Request) bool { + if request.URL.Path != callbackPath || !constantTimeEqual(request.Host, strings.TrimPrefix(f.callbackURLHost(), "//")) { + return false + } + query := request.URL.Query() + states, ok := query["state"] + return ok && len(query) == 1 && len(states) == 1 && constantTimeEqual(states[0], f.state) +} + +func (f *browserFlow) callbackURLHost() string { + parsed, err := url.Parse(f.callbackURL) + if err != nil { + return "" + } + return parsed.Host +} + +func (f *browserFlow) setCORSHeaders(header http.Header) { + header.Set("Access-Control-Allow-Origin", f.origin) + header.Set("Access-Control-Allow-Methods", "POST") + header.Set("Access-Control-Allow-Headers", "Content-Type") + header.Set("Access-Control-Allow-Private-Network", "true") + header.Add("Vary", "Origin") + header.Add("Vary", "Access-Control-Request-Method") + header.Add("Vary", "Access-Control-Request-Headers") +} + +func allowsContentTypeHeader(value string) bool { + for _, part := range strings.Split(value, ",") { + if strings.EqualFold(strings.TrimSpace(part), "content-type") { + return true + } + } + return false +} + +func ensureJSONEOF(decoder *json.Decoder) error { + var trailing any + err := decoder.Decode(&trailing) + if errors.Is(err, io.EOF) { + return nil + } + if err == nil { + return errors.New("unexpected trailing JSON") + } + return err +} + +func originOf(value *url.URL) string { + return fmt.Sprintf("%s://%s", strings.ToLower(value.Scheme), value.Host) +} From 0e96f8cf092cf165e793c719223f4919466e7c9b Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 07:15:45 +0800 Subject: [PATCH 38/48] feat(auth): provision managed access keys in browser login Co-authored-by: Codex <codex@openai.com> --- internal/auth/api.go | 299 ++++++++++++++++++++++++ internal/auth/manager.go | 485 +++++++++++++++++++++++++++------------ 2 files changed, 640 insertions(+), 144 deletions(-) create mode 100644 internal/auth/api.go diff --git a/internal/auth/api.go b/internal/auth/api.go new file mode 100644 index 0000000..a432657 --- /dev/null +++ b/internal/auth/api.go @@ -0,0 +1,299 @@ +package auth + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/cookiejar" + "net/url" + "strconv" + "strings" +) + +const ( + maxAPIResponseBytes = 1 << 20 + loginGrantScope = "xyq_novel_cli_login" +) + +type apiEnvelope[T any] struct { + Ret json.RawMessage `json:"ret"` + Errmsg string `json:"errmsg"` + Data T `json:"data"` +} + +type apiResponseError struct { + operation string + httpStatus int + ret string +} + +func (err *apiResponseError) Error() string { + if err.httpStatus != 0 { + return fmt.Sprintf("%s失败（HTTP %d）", err.operation, err.httpStatus) + } + return fmt.Sprintf("%s失败（服务端错误码 %s）", err.operation, err.ret) +} + +type exchangeData struct { + UID string `json:"uid"` + Scope string `json:"scope"` +} + +type queryAccessKeyData struct { + AccessTokens []accessToken `json:"access_token_list"` +} + +type accessToken struct { + ID string `json:"ak_id"` + Token string `json:"token"` + ExpiredAt int64 `json:"expired_at"` + Name string `json:"token_name"` + Status string `json:"token_status"` +} + +type generateAccessKeyData struct { + AccessKey string `json:"ak"` + TokenID string `json:"token_id"` +} + +func (m *Manager) exchangeAndProvision( + ctx context.Context, + payload loginGrantPayload, + identity *Credential, + options LoginOptions, +) (*Credential, error) { + if payload.ExpireAt > 0 && payload.ExpireAt <= m.now().Unix() { + return nil, errors.New("网页授权已过期，请重新登录") + } + jar, err := cookiejar.New(nil) + if err != nil { + return nil, errors.New("初始化临时网页登录会话失败") + } + client := *m.httpClient + client.Jar = jar + client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + } + + exchangeRequest := struct { + Grant string `json:"grant"` + Secret string `json:"random_secret_key"` + }{Grant: payload.Grant, Secret: payload.RandomSecretKey} + exchange, err := doJSON[exchangeData](ctx, &client, m.authBaseURL, exchangeGrantPath, exchangeRequest, "交换网页授权") + if err != nil { + return nil, err + } + if strings.TrimSpace(exchange.UID) == "" || !constantTimeEqual(exchange.Scope, loginGrantScope) { + return nil, errors.New("网页授权响应缺少有效的用户身份") + } + exchange.UID = strings.TrimSpace(exchange.UID) + actualScope := credentialScope(exchange.UID, identity.DeviceID) + if expected := strings.TrimSpace(options.ExpectedCredentialScope); expected != "" && + !constantTimeEqual(expected, actualScope) { + return nil, ErrCredentialAccountMismatch + } + if len(jar.Cookies(m.authBaseURL)) == 0 { + return nil, errors.New("网页授权响应没有建立临时登录会话") + } + + query, err := doJSON[queryAccessKeyData](ctx, &client, m.authBaseURL, queryAccessKeyPath, nil, "查询 CLI 凭证") + if err != nil { + return nil, err + } + var rotatedTokenIDs []string + if options.ForceRefresh { + // A stored TokenID only identifies this device's token inside the + // account that originally issued it. Never use it as a destructive + // selector after the browser has switched to another UID. + if identity.UID != "" && constantTimeEqual(identity.UID, exchange.UID) { + rotatedTokenIDs = managedTokenIDs(query.AccessTokens, identity) + } + if len(rotatedTokenIDs) > 0 { + deleteRequest := struct { + AKIDs []string `json:"ak_ids"` + }{AKIDs: rotatedTokenIDs} + if _, err := doJSON[struct{}](ctx, &client, m.authBaseURL, deleteAccessKeyPath, deleteRequest, "轮换旧的 CLI 凭证"); err != nil { + return nil, err + } + } + } else { + selected, err := m.selectManagedToken(query.AccessTokens, identity) + if err != nil { + return nil, err + } + if selected != nil { + return credentialFromToken(identity, exchange.UID, selected), nil + } + } + + expiredAt := m.now().Add(DefaultCredentialLifetime).Unix() + generateRequest := struct { + TokenName string `json:"token_name"` + TokenDesc string `json:"token_desc"` + ExpiredAt int64 `json:"expired_at"` + }{ + TokenName: identity.TokenName, + TokenDesc: "Pippit Tool CLI browser login", + ExpiredAt: expiredAt, + } + generated, err := doJSON[generateAccessKeyData](ctx, &client, m.authBaseURL, generateAccessKeyPath, generateRequest, "创建 CLI 凭证") + if err != nil { + var responseErr *apiResponseError + if errors.As(err, &responseErr) && responseErr.ret == "3" { + return nil, errors.New("当前账号暂不具备创建 CLI Access Key 的权限，请升级、联系管理员或使用已有 XYQ_ACCESS_KEY") + } + if errors.As(err, &responseErr) && responseErr.ret != "" { + return nil, errors.New("无法创建新的 CLI Access Key；请在个人设置中检查 Access Key 数量上限和账号权限后重试") + } + return nil, err + } + if strings.TrimSpace(generated.AccessKey) == "" || strings.TrimSpace(generated.TokenID) == "" { + return nil, errors.New("创建 CLI 凭证后服务端未返回完整结果") + } + credential := cloneCredential(identity) + credential.AccessKey = strings.TrimSpace(generated.AccessKey) + credential.TokenID = strings.TrimSpace(generated.TokenID) + credential.UID = exchange.UID + credential.CredentialScope = actualScope + credential.ExpiredAt = expiredAt + if options.ForceRefresh && identity.UID == credential.UID && identity.AccessKey != "" && + constantTimeEqual(identity.AccessKey, credential.AccessKey) { + return nil, errors.New("服务端未轮换已失效的 CLI Access Key，已拒绝继续使用旧凭证") + } + for _, tokenID := range rotatedTokenIDs { + if constantTimeEqual(tokenID, credential.TokenID) { + return nil, errors.New("服务端未轮换已失效的 CLI 凭证编号，已拒绝继续使用旧凭证") + } + } + if err := validateCredential(credential); err != nil { + return nil, errors.New("创建的 CLI 凭证格式无效") + } + return credential, nil +} + +func managedTokenIDs(tokens []accessToken, identity *Credential) []string { + result := make([]string, 0, 1) + if identity == nil || strings.TrimSpace(identity.TokenID) == "" { + return result + } + for _, token := range tokens { + id := strings.TrimSpace(token.ID) + // QueryAk is scoped by the exchanged browser UID, while TokenID comes + // from this device's securely stored credential. Their exact match is + // the destructive-operation boundary even if the user renamed the token. + if id == "" || !constantTimeEqual(id, identity.TokenID) { + continue + } + result = append(result, id) + break + } + return result +} + +func (m *Manager) selectManagedToken(tokens []accessToken, identity *Credential) (*accessToken, error) { + valid := make([]accessToken, 0, 1) + minimumExpiry := m.now().Add(m.ensureTTL()).Unix() + if identity.TokenID != "" { + for index := range tokens { + if constantTimeEqual(tokens[index].ID, identity.TokenID) && usableAccessToken(tokens[index], minimumExpiry) { + // TokenID is the stable device-owned identity. Prefer it before + // matching the display name because users may rename a token in UI. + return &tokens[index], nil + } + } + } + for _, token := range tokens { + if !constantTimeEqual(token.Name, identity.TokenName) || !usableAccessToken(token, minimumExpiry) { + continue + } + valid = append(valid, token) + } + if len(valid) == 0 { + return nil, nil + } + if len(valid) != 1 { + return nil, errors.New("检测到多个同设备 CLI 凭证，拒绝自动选择；请在个人设置中清理重复项后重试") + } + return &valid[0], nil +} + +func usableAccessToken(token accessToken, minimumExpiry int64) bool { + return token.Status == "enable" && strings.TrimSpace(token.Token) != "" && token.ExpiredAt > minimumExpiry +} + +func credentialFromToken(identity *Credential, uid string, token *accessToken) *Credential { + credential := cloneCredential(identity) + credential.AccessKey = strings.TrimSpace(token.Token) + credential.TokenID = strings.TrimSpace(token.ID) + credential.UID = strings.TrimSpace(uid) + credential.CredentialScope = credentialScope(credential.UID, credential.DeviceID) + credential.ExpiredAt = token.ExpiredAt + return credential +} + +func doJSON[T any](ctx context.Context, client *http.Client, baseURL *url.URL, path string, body any, operation string) (T, error) { + var zero T + requestURL := *baseURL + requestURL.Path = path + requestURL.RawPath = "" + requestURL.RawQuery = "" + requestURL.Fragment = "" + + var reader io.Reader + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return zero, redactedOperationError(operation) + } + reader = bytes.NewReader(payload) + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL.String(), reader) + if err != nil { + return zero, redactedOperationError(operation) + } + request.Header.Set("Accept", "application/json") + request.Header.Set("User-Agent", "Pippit-CLI/1.0") + request.Header.Set("appvr", "1.1.4") + request.Header.Set("entrance-from", "web") + request.Header.Set("appid", "795647") + if body != nil { + request.Header.Set("Content-Type", "application/json") + } + response, err := client.Do(request) + if err != nil { + return zero, redactedOperationError(operation) + } + defer response.Body.Close() + responseBody, err := io.ReadAll(io.LimitReader(response.Body, maxAPIResponseBytes+1)) + if err != nil || len(responseBody) > maxAPIResponseBytes { + return zero, redactedOperationError(operation) + } + if response.StatusCode < 200 || response.StatusCode >= 300 { + return zero, &apiResponseError{operation: operation, httpStatus: response.StatusCode} + } + envelope := apiEnvelope[T]{} + if err := json.Unmarshal(responseBody, &envelope); err != nil { + return zero, fmt.Errorf("%s响应格式无效", operation) + } + if !successfulRet(envelope.Ret) { + return zero, &apiResponseError{operation: operation, ret: safeRet(envelope.Ret)} + } + return envelope.Data, nil +} + +func successfulRet(value json.RawMessage) bool { + trimmed := strings.TrimSpace(string(value)) + return trimmed == "" || trimmed == "null" || trimmed == `""` || trimmed == "0" || trimmed == `"0"` +} + +func safeRet(value json.RawMessage) string { + trimmed := strings.Trim(strings.TrimSpace(string(value)), `"`) + if _, err := strconv.ParseInt(trimmed, 10, 64); err == nil && len(trimmed) <= 20 { + return trimmed + } + return "unknown" +} diff --git a/internal/auth/manager.go b/internal/auth/manager.go index 71eaede..8a9271d 100644 --- a/internal/auth/manager.go +++ b/internal/auth/manager.go @@ -1,146 +1,343 @@ package auth -//import ( -// "context" -// "errors" -// "fmt" -// "net/http" -// "sync" -// "time" -// -// "code.byted.org/passport/auth_client/go/authsdk" -// -// "github.com/Pippit-dev/pippit-cli/internal/config" -//) -// -//type Authorizer interface { -// Refresh(ctx context.Context, ensureTTL time.Duration) error -// Inject(ctx context.Context, req *http.Request) error -// NewLoginFlow(ctx context.Context) (*LoginFlow, error) -// CheckLogin(ctx context.Context, deviceCode string) (*State, error) -// State(ctx context.Context) (*State, error) -// Logout(ctx context.Context) error -//} -// -//type OAuthManager struct { -// cfg *config.Config -// mu sync.Mutex -// client *authsdk.Client -//} -// -//type LoginFlow struct { -// DeviceCode string `json:"device_code"` -// UserCode string `json:"user_code"` -// VerificationURI string `json:"verification_uri"` -//} -// -//type State struct { -// LoggedIn bool `json:"logged_in"` -// ExpiresAt time.Time `json:"expires_at,omitempty"` -//} -// -//func NewManager(cfg *config.Config) *OAuthManager { -// return &OAuthManager{cfg: cfg} -//} -// -//func (m *OAuthManager) NewLoginFlow(ctx context.Context) (*LoginFlow, error) { -// client, err := m.clientInstance() -// if err != nil { -// return nil, err -// } -// flow, err := client.Authenticator().NewLoginFlow(ctx) -// if err != nil { -// return nil, err -// } -// return &LoginFlow{ -// DeviceCode: flow.DeviceCode, -// UserCode: flow.UserCode, -// VerificationURI: flow.VerificationURI, -// }, nil -//} -// -//func (m *OAuthManager) CheckLogin(ctx context.Context, deviceCode string) (*State, error) { -// client, err := m.clientInstance() -// if err != nil { -// return nil, err -// } -// state, err := client.Authenticator().CheckLogin(ctx, deviceCode) -// if err != nil { -// return nil, err -// } -// return authState(state), nil -//} -// -//func (m *OAuthManager) State(ctx context.Context) (*State, error) { -// client, err := m.clientInstance() -// if err != nil { -// return nil, err -// } -// state, err := client.Authorizer().State(ctx) -// if err != nil { -// return nil, err -// } -// return authState(state), nil -//} -// -//func (m *OAuthManager) Refresh(ctx context.Context, ensureTTL time.Duration) error { -// client, err := m.clientInstance() -// if err != nil { -// return err -// } -// _, err = client.Authorizer().Refresh(ctx, ensureTTL) -// return err -//} -// -//func (m *OAuthManager) Inject(ctx context.Context, req *http.Request) error { -// client, err := m.clientInstance() -// if err != nil { -// return err -// } -// return client.Authorizer().Inject(ctx, req) -//} -// -//func (m *OAuthManager) Logout(ctx context.Context) error { -// client, err := m.clientInstance() -// if err != nil { -// return err -// } -// return client.Authorizer().Logout(ctx) -//} -// -//func IsLoginPending(err error) bool { -// return errors.Is(err, authsdk.ErrLoginPending) -//} -// -//func (m *OAuthManager) clientInstance() (*authsdk.Client, error) { -// m.mu.Lock() -// defer m.mu.Unlock() -// if m.client != nil { -// return m.client, nil -// } -// if m.cfg == nil || m.cfg.OAuth == nil { -// return nil, errors.New("oauth config is required") -// } -// oauth := m.cfg.OAuth -// client, err := authsdk.NewClient(authsdk.Config{ -// ClientKey: oauth.ClientKey, -// BaseURL: oauth.BaseURL, -// StoreServiceName: oauth.StoreServiceName, -// Scopes: oauth.Scopes, -// }) -// if err != nil { -// return nil, fmt.Errorf("initialize oauth client: %w", err) -// } -// m.client = client -// return client, nil -//} -// -//func authState(state *authsdk.AuthStateView) *State { -// if state == nil { -// return &State{} -// } -// return &State{ -// LoggedIn: state.LoggedIn, -// ExpiresAt: state.ExpiresAt, -// } -//} +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/Pippit-dev/pippit-cli/internal/config" +) + +type Manager struct { + cfg *config.Config + store CredentialStore + httpClient *http.Client + authBaseURL *url.URL + random io.Reader + now func() time.Time + credentialMu sync.Mutex + cachedCredential *Credential + credentialCacheLoaded bool +} + +type ManagerOption func(*Manager) + +func WithCredentialStore(store CredentialStore) ManagerOption { + return func(manager *Manager) { + if store != nil { + manager.store = store + } + } +} + +func WithHTTPClient(client *http.Client) ManagerOption { + return func(manager *Manager) { + if client != nil { + manager.httpClient = client + } + } +} + +func withAuthBaseURLForTest(rawURL string) ManagerOption { + return func(manager *Manager) { + parsed, err := url.Parse(rawURL) + if err == nil { + manager.authBaseURL = parsed + } + } +} + +func withRandomReaderForTest(reader io.Reader) ManagerOption { + return func(manager *Manager) { + if reader != nil { + manager.random = reader + } + } +} + +func withClockForTest(now func() time.Time) ManagerOption { + return func(manager *Manager) { + if now != nil { + manager.now = now + } + } +} + +// NewManager always uses the canonical production auth origin. cfg.BaseURL and +// cfg.PPEEnv intentionally do not affect browser grants, login cookies, or AK +// provisioning; PPE routing applies only after a dedicated AK has been issued. +func NewManager(cfg *config.Config, options ...ManagerOption) *Manager { + serviceName := config.DefaultAuthStoreServiceName + authBaseURL, _ := url.Parse(config.DefaultBaseURL) + timeout := config.DefaultHTTPTimeout + if cfg != nil && cfg.HTTPTimeout > 0 { + timeout = cfg.HTTPTimeout + } + manager := &Manager{ + cfg: cfg, + store: NewDefaultCredentialStore(serviceName), + httpClient: &http.Client{Timeout: timeout}, + authBaseURL: authBaseURL, + random: rand.Reader, + now: time.Now, + } + for _, option := range options { + if option != nil { + option(manager) + } + } + return manager +} + +func (m *Manager) ResolveAccessKey(ctx context.Context) (string, error) { + if m != nil && m.cfg != nil { + if accessKey := strings.TrimSpace(m.cfg.AccessKey); accessKey != "" { + return accessKey, nil + } + } + credential, err := m.loadCredential(ctx) + if err != nil { + return "", err + } + if credential.AccessKey == "" { + return "", ErrCredentialNotFound + } + if credential.ExpiredAt <= m.now().Add(m.ensureTTL()).Unix() { + return "", ErrCredentialExpired + } + return credential.AccessKey, nil +} + +func (m *Manager) Login(ctx context.Context, options LoginOptions) (*Credential, error) { + if err := m.validate(); err != nil { + return nil, err + } + identity, err := m.ensureIdentity(ctx) + if err != nil { + return nil, err + } + flow, err := startBrowserFlow(m.authBaseURL, m.random) + if err != nil { + return nil, err + } + defer flow.close() + + writeProgress(options.Progress, "正在打开小云雀网页授权…") + opener := options.OpenURL + if opener == nil { + opener = OpenBrowser + } + if err := opener(flow.loginURL); err != nil { + return nil, errors.New("无法自动打开浏览器，请检查系统默认浏览器设置后重试") + } + writeProgress(options.Progress, "请在浏览器中完成登录和授权，CLI 会自动继续…") + + timeout := options.Timeout + if timeout <= 0 { + timeout = DefaultLoginTimeout + } + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + payload, err := flow.wait(waitCtx) + if err != nil { + return nil, err + } + writeProgress(options.Progress, "网页授权已完成，正在准备本机专属 CLI 凭证…") + credential, err := m.exchangeAndProvision(waitCtx, payload, identity, options) + if err != nil { + return nil, err + } + if err := m.saveCredential(waitCtx, credential); err != nil { + return nil, err + } + writeProgress(options.Progress, "小云雀 CLI 登录成功。") + return cloneCredential(credential), nil +} + +func (m *Manager) Status(ctx context.Context) (*Status, error) { + if m != nil && m.cfg != nil && strings.TrimSpace(m.cfg.AccessKey) != "" { + return &Status{LoggedIn: true, Source: "environment"}, nil + } + credential, err := m.loadCredential(ctx) + if errors.Is(err, ErrCredentialNotFound) { + return &Status{}, nil + } + if err != nil { + return nil, err + } + status := &Status{ + Source: "browser", + UID: credential.UID, + TokenID: credential.TokenID, + CredentialScope: credential.CredentialScope, + } + if credential.ExpiredAt > 0 { + status.ExpiresAt = time.Unix(credential.ExpiredAt, 0) + } + status.LoggedIn = credential.AccessKey != "" && credential.ExpiredAt > m.now().Add(m.ensureTTL()).Unix() + return status, nil +} + +func (m *Manager) Logout(ctx context.Context, revoke bool) error { + if revoke { + // The current AK management endpoint requires a browser/team session and + // cannot be safely called using the Access Key that would be revoked. + return ErrRemoteRevokeUnsupported + } + credential, err := m.loadCredential(ctx) + if errors.Is(err, ErrCredentialNotFound) { + m.clearCredentialCache() + return nil + } + if err != nil { + return err + } + // Keep the non-secret device identity so a later login can safely reuse the + // same remote token instead of consuming another per-user AK slot. Logout + // only clears the local secret and account binding; it does not revoke the + // remote Access Key. + if err := m.saveCredential(ctx, identityOnly(credential)); err != nil { + return err + } + m.clearCredentialCache() + return nil +} + +func (m *Manager) CredentialScope(ctx context.Context) (string, error) { + credential, err := m.loadCredential(ctx) + if err != nil { + return "", err + } + if credential.AccessKey == "" || strings.TrimSpace(credential.UID) == "" { + return "", ErrCredentialNotFound + } + return credentialScope(credential.UID, credential.DeviceID), nil +} + +func (m *Manager) ensureIdentity(ctx context.Context) (*Credential, error) { + credential, err := m.loadCredential(ctx) + if err == nil { + return credential, nil + } + if !errors.Is(err, ErrCredentialNotFound) { + return nil, err + } + identity, err := newIdentity(m.random) + if err != nil { + return nil, err + } + if err := m.saveCredential(ctx, identity); err != nil { + return nil, err + } + return identity, nil +} + +func (m *Manager) loadCredential(ctx context.Context) (*Credential, error) { + if err := m.validate(); err != nil { + return nil, err + } + m.credentialMu.Lock() + defer m.credentialMu.Unlock() + if m.credentialCacheLoaded { + if m.cachedCredential == nil { + return nil, ErrCredentialNotFound + } + return cloneCredential(m.cachedCredential), nil + } + credential, err := m.store.Load(ctx) + if err != nil { + if errors.Is(err, ErrCredentialNotFound) { + m.credentialCacheLoaded = true + m.cachedCredential = nil + } + return nil, err + } + credential = normalizeCredential(credential) + m.cachedCredential = cloneCredential(credential) + m.credentialCacheLoaded = true + return cloneCredential(credential), nil +} + +func (m *Manager) saveCredential(ctx context.Context, credential *Credential) error { + if err := m.validate(); err != nil { + return err + } + credential = normalizeCredential(credential) + if err := m.store.Save(ctx, credential); err != nil { + m.clearCredentialCache() + return err + } + m.credentialMu.Lock() + m.cachedCredential = cloneCredential(credential) + m.credentialCacheLoaded = true + m.credentialMu.Unlock() + return nil +} + +func (m *Manager) clearCredentialCache() { + if m == nil { + return + } + m.credentialMu.Lock() + m.cachedCredential = nil + m.credentialCacheLoaded = false + m.credentialMu.Unlock() +} + +func normalizeCredential(credential *Credential) *Credential { + credential = cloneCredential(credential) + if credential == nil { + return nil + } + if credential.AccessKey == "" { + credential.CredentialScope = "" + return credential + } + credential.CredentialScope = credentialScope(credential.UID, credential.DeviceID) + return credential +} + +func (m *Manager) validate() error { + if m == nil || m.store == nil || m.httpClient == nil || m.authBaseURL == nil || m.random == nil || m.now == nil { + return errors.New("小云雀 CLI 授权管理器未正确初始化") + } + if m.authBaseURL.Scheme != "https" && !(m.authBaseURL.Scheme == "http" && isLoopbackHost(m.authBaseURL.Hostname())) { + return errors.New("小云雀授权地址必须使用 HTTPS") + } + if m.authBaseURL.Host == "" || m.authBaseURL.RawQuery != "" || m.authBaseURL.Fragment != "" { + return errors.New("小云雀授权地址无效") + } + return nil +} + +func (m *Manager) ensureTTL() time.Duration { + if m != nil && m.cfg != nil && m.cfg.AuthTTL > 0 { + return m.cfg.AuthTTL + } + return config.DefaultAuthTTL +} + +func isLoopbackHost(host string) bool { + return host == "127.0.0.1" || strings.EqualFold(host, "localhost") || host == "::1" +} + +func writeProgress(writer io.Writer, message string) { + if writer != nil { + _, _ = fmt.Fprintln(writer, message) + } +} + +func cloneCredential(credential *Credential) *Credential { + if credential == nil { + return nil + } + copy := *credential + return © +} From b22b9ee5bc958b777c5e580ccbe9e8f830731287 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 07:18:02 +0800 Subject: [PATCH 39/48] test(auth): cover secure browser credentials Co-authored-by: Codex <codex@openai.com> --- internal/auth/auth_test.go | 877 +++++++++++++++++++++++++++++++++++++ 1 file changed, 877 insertions(+) create mode 100644 internal/auth/auth_test.go diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go new file mode 100644 index 0000000..3b435a4 --- /dev/null +++ b/internal/auth/auth_test.go @@ -0,0 +1,877 @@ +package auth + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Pippit-dev/pippit-cli/internal/config" +) + +type memoryCredentialStore struct { + mu sync.Mutex + credential *Credential + loadErr error + saveErr error + deleteErr error + loads int + saves int + deletes int +} + +func (s *memoryCredentialStore) Load(ctx context.Context) (*Credential, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + s.mu.Lock() + defer s.mu.Unlock() + s.loads++ + if s.loadErr != nil { + return nil, s.loadErr + } + if s.credential == nil { + return nil, ErrCredentialNotFound + } + return cloneCredential(s.credential), nil +} + +func (s *memoryCredentialStore) Save(ctx context.Context, credential *Credential) error { + if err := ctx.Err(); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + s.saves++ + if s.saveErr != nil { + return s.saveErr + } + s.credential = cloneCredential(credential) + return nil +} + +func (s *memoryCredentialStore) Delete(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + s.deletes++ + if s.deleteErr != nil { + return s.deleteErr + } + if s.credential == nil { + return ErrCredentialNotFound + } + s.credential = nil + return nil +} + +func TestBrowserFlowRequiresExactBoundCallbackAndCORS(t *testing.T) { + authURL, _ := url.Parse("https://xyq.jianying.com") + flow, err := startBrowserFlow(authURL, bytes.NewReader(bytes.Repeat([]byte{0x42}, 64))) + if err != nil { + t.Fatal(err) + } + defer flow.close() + + loginURL, err := url.Parse(flow.loginURL) + if err != nil { + t.Fatal(err) + } + if got := loginURL.Query().Get("source"); got != loginSource { + t.Fatalf("source = %q, want %q", got, loginSource) + } + if got := loginURL.Query().Get("ppe_env"); got != "" { + t.Fatalf("ppe_env = %q, want absent", got) + } + for _, name := range []string{"random_secret_key"} { + decoded, err := base64.RawURLEncoding.DecodeString(loginURL.Query().Get(name)) + if err != nil || len(decoded) < randomBindingBytes { + t.Fatalf("%s is not at least %d random bytes", name, randomBindingBytes) + } + } + callback, err := url.Parse(flow.callbackURL) + if err != nil { + t.Fatal(err) + } + decodedState, err := base64.RawURLEncoding.DecodeString(callback.Query().Get("state")) + if err != nil || len(decodedState) < randomBindingBytes { + t.Fatalf("state is not at least %d random bytes", randomBindingBytes) + } + + preflight, _ := http.NewRequest(http.MethodOptions, flow.callbackURL, nil) + preflight.Header.Set("Origin", "https://xyq.jianying.com") + preflight.Header.Set("Access-Control-Request-Method", http.MethodPost) + preflight.Header.Set("Access-Control-Request-Headers", "content-type") + preflightResponse, err := http.DefaultClient.Do(preflight) + if err != nil { + t.Fatal(err) + } + preflightResponse.Body.Close() + if preflightResponse.StatusCode != http.StatusNoContent { + t.Fatalf("preflight status = %d", preflightResponse.StatusCode) + } + if got := preflightResponse.Header.Get("Access-Control-Allow-Origin"); got != "https://xyq.jianying.com" { + t.Fatalf("allow origin = %q", got) + } + if got := preflightResponse.Header.Get("Access-Control-Allow-Private-Network"); got != "true" { + t.Fatalf("allow private network = %q", got) + } + + payload := loginGrantPayload{ + Type: "login_grant", + Grant: "one-time-grant", + RandomSecretKey: flow.secret, + Source: flow.source, + CallbackURL: flow.callbackURL, + } + wrong := payload + wrong.Source = "other-cli" + if status := postCallback(t, flow.callbackURL, flow.origin, wrong); status != http.StatusBadRequest { + t.Fatalf("wrong binding status = %d", status) + } + wrongStateURL := *callback + wrongStateQuery := wrongStateURL.Query() + wrongStateQuery.Set("state", "wrong-state") + wrongStateURL.RawQuery = wrongStateQuery.Encode() + if status := postCallback(t, wrongStateURL.String(), flow.origin, payload); status != http.StatusBadRequest { + t.Fatalf("wrong state status = %d", status) + } + if status := postCallback(t, flow.callbackURL, "https://attacker.invalid", payload); status != http.StatusForbidden { + t.Fatalf("wrong origin status = %d", status) + } + if status := postCallback(t, flow.callbackURL, flow.origin, payload); status != http.StatusOK { + t.Fatalf("valid callback status = %d", status) + } + got, err := flow.wait(context.Background()) + if err != nil { + t.Fatal(err) + } + if got.Grant != payload.Grant { + t.Fatalf("grant = %q", got.Grant) + } +} + +func TestManagerLoginReusesOnlyExactDeviceToken(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + store := &memoryCredentialStore{} + var generated bool + var expectedName string + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + assertAuthHeaders(t, request) + if request.Header.Get("x-use-ppe") != "" || request.Header.Get("x-tt-env") != "" { + t.Errorf("auth request unexpectedly carried PPE headers") + } + switch request.URL.Path { + case exchangeGrantPath: + var body map[string]string + _ = json.NewDecoder(request.Body).Decode(&body) + if body["grant"] != "grant-value" || body["random_secret_key"] == "" { + t.Errorf("unexpected exchange body") + } + http.SetCookie(writer, &http.Cookie{Name: "session", Value: "cookie-secret", Path: "/", Secure: true, HttpOnly: true}) + writeEnvelope(writer, map[string]any{"uid": "123", "scope": loginGrantScope}) + case queryAccessKeyPath: + cookie, err := request.Cookie("session") + if err != nil || cookie.Value != "cookie-secret" { + t.Errorf("query did not receive temporary exchange cookie") + } + credential, err := store.Load(context.Background()) + if err != nil { + t.Errorf("load identity: %v", err) + return + } + expectedName = credential.TokenName + writeEnvelope(writer, map[string]any{"access_token_list": []map[string]any{ + {"ak_id": "foreign-id", "token": "foreign-ak", "expired_at": fixedNow.Add(time.Hour).Unix(), "token_name": "someone-else", "token_status": "enable"}, + {"ak_id": "managed-id", "token": "managed-ak", "expired_at": fixedNow.Add(time.Hour).Unix(), "token_name": credential.TokenName, "token_status": "enable"}, + }}) + case generateAccessKeyPath: + generated = true + http.Error(writer, "must not generate", http.StatusInternalServerError) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + + cfg := config.Load() + cfg.AccessKey = "" + cfg.PPEEnv = "ppe_must_not_reach_auth" + manager := NewManager(cfg, + WithCredentialStore(store), + WithHTTPClient(server.Client()), + withAuthBaseURLForTest(server.URL), + withClockForTest(func() time.Time { return fixedNow }), + ) + var progress bytes.Buffer + callbackResult := make(chan error, 1) + credential, err := manager.Login(context.Background(), LoginOptions{ + Timeout: 2 * time.Second, + Progress: &progress, + OpenURL: func(rawURL string) error { + loginURL, err := url.Parse(rawURL) + if err != nil { + return err + } + if loginURL.Scheme != "https" || loginURL.Host != strings.TrimPrefix(server.URL, "https://") { + t.Errorf("login URL origin = %s://%s", loginURL.Scheme, loginURL.Host) + } + if loginURL.Query().Get("ppe_env") != "" { + t.Error("login URL carried PPE lane") + } + payload := loginGrantPayload{ + Type: "login_grant", + Grant: "grant-value", + RandomSecretKey: loginURL.Query().Get("random_secret_key"), + Source: loginSource, + CallbackURL: loginURL.Query().Get("callback"), + } + go func() { + status, err := sendCallback(payload.CallbackURL, server.URL, payload) + if err == nil && status != http.StatusOK { + err = errors.New("callback did not return HTTP 200") + } + callbackResult <- err + }() + return nil + }, + }) + if err != nil { + t.Fatal(err) + } + if err := <-callbackResult; err != nil { + t.Fatal(err) + } + if generated { + t.Fatal("generate endpoint was called despite exact valid token") + } + if credential.AccessKey != "managed-ak" || credential.TokenID != "managed-id" || credential.UID != "123" { + t.Fatalf("credential metadata = %#v", credentialWithoutSecret(credential)) + } + if credential.TokenName != expectedName || len(credential.TokenName) > 48 { + t.Fatalf("token name = %q", credential.TokenName) + } + publicJSON, err := json.Marshal(credential) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(publicJSON), credential.AccessKey) { + t.Fatal("credential JSON exposed the Access Key") + } + if strings.Contains(progress.String(), "managed-ak") || strings.Contains(progress.String(), "grant-value") || strings.Contains(progress.String(), "cookie-secret") { + t.Fatalf("progress leaked credentials: %q", progress.String()) + } +} + +func TestExchangeGeneratesWhenExactValidTokenDoesNotExist(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + identity, err := newIdentity(bytes.NewReader(bytes.Repeat([]byte{7}, deviceIDBytes))) + if err != nil { + t.Fatal(err) + } + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + assertAuthHeaders(t, request) + switch request.URL.Path { + case exchangeGrantPath: + http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) + writeEnvelope(writer, map[string]any{"uid": "456", "scope": loginGrantScope}) + case queryAccessKeyPath: + writeEnvelope(writer, map[string]any{"access_token_list": []map[string]any{ + {"ak_id": "foreign-id", "token": "foreign-ak", "expired_at": fixedNow.Add(time.Hour).Unix(), "token_name": "foreign", "token_status": "enable"}, + {"ak_id": "expired-id", "token": "expired-ak", "expired_at": fixedNow.Add(-time.Hour).Unix(), "token_name": identity.TokenName, "token_status": "enable"}, + }}) + case generateAccessKeyPath: + var body struct { + TokenName string `json:"token_name"` + ExpiredAt int64 `json:"expired_at"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Error(err) + } + if body.TokenName != identity.TokenName { + t.Errorf("generated token name = %q", body.TokenName) + } + if body.ExpiredAt != fixedNow.Add(DefaultCredentialLifetime).Unix() { + t.Errorf("generated expiry = %d", body.ExpiredAt) + } + writeEnvelope(writer, map[string]any{"ak": "new-managed-ak", "token_id": "new-managed-id"}) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + manager := NewManager(config.Load(), + WithCredentialStore(&memoryCredentialStore{}), + WithHTTPClient(server.Client()), + withAuthBaseURLForTest(server.URL), + withClockForTest(func() time.Time { return fixedNow }), + ) + credential, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ + Grant: "grant", RandomSecretKey: "secret", ExpireAt: fixedNow.Add(time.Minute).Unix(), + }, identity, LoginOptions{}) + if err != nil { + t.Fatal(err) + } + if credential.AccessKey != "new-managed-ak" || credential.TokenID != "new-managed-id" || credential.UID != "456" { + t.Fatalf("credential metadata = %#v", credentialWithoutSecret(credential)) + } +} + +func TestForceRefreshDeletesExactDeviceTokensBeforeGeneratingReplacement(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{8}, deviceIDBytes))) + identity.AccessKey = "rejected-ak" + identity.TokenID = "rejected-id" + identity.UID = "456" + identity.CredentialScope = credentialScope(identity.UID, identity.DeviceID) + identity.ExpiredAt = fixedNow.Add(time.Hour).Unix() + + var calls []string + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + calls = append(calls, request.URL.Path) + switch request.URL.Path { + case exchangeGrantPath: + http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) + writeEnvelope(writer, map[string]any{"uid": identity.UID, "scope": loginGrantScope}) + case queryAccessKeyPath: + writeEnvelope(writer, map[string]any{"access_token_list": []map[string]any{ + {"ak_id": identity.TokenID, "token": identity.AccessKey, "expired_at": identity.ExpiredAt, "token_name": "user-renamed-token", "token_status": "enable"}, + {"ak_id": "stale-duplicate", "token": "stale-ak", "expired_at": identity.ExpiredAt, "token_name": identity.TokenName, "token_status": "disable"}, + {"ak_id": "foreign-id", "token": "foreign-ak", "expired_at": identity.ExpiredAt, "token_name": "another-device", "token_status": "enable"}, + }}) + case deleteAccessKeyPath: + var body struct { + AKIDs []string `json:"ak_ids"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Error(err) + } + if strings.Join(body.AKIDs, ",") != "rejected-id" { + t.Errorf("deleted IDs = %v", body.AKIDs) + } + writeEnvelope(writer, map[string]any{}) + case generateAccessKeyPath: + writeEnvelope(writer, map[string]any{"ak": "replacement-ak", "token_id": "replacement-id"}) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + + manager := NewManager(config.Load(), WithHTTPClient(server.Client()), withAuthBaseURLForTest(server.URL), withClockForTest(func() time.Time { return fixedNow })) + credential, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ + Grant: "grant", RandomSecretKey: "secret", ExpireAt: fixedNow.Add(time.Minute).Unix(), + }, identity, LoginOptions{ForceRefresh: true, ExpectedCredentialScope: identity.CredentialScope}) + if err != nil { + t.Fatal(err) + } + if credential.AccessKey != "replacement-ak" || credential.TokenID != "replacement-id" { + t.Fatalf("replacement credential = %#v", credentialWithoutSecret(credential)) + } + if got := strings.Join(calls, ","); got != exchangeGrantPath+","+queryAccessKeyPath+","+deleteAccessKeyPath+","+generateAccessKeyPath { + t.Fatalf("endpoint order = %q", got) + } +} + +func TestForceRefreshRejectsDifferentBrowserAccountBeforeAKMutation(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{6}, deviceIDBytes))) + identity.AccessKey = "account-a-ak" + identity.TokenID = "account-a-id" + identity.UID = "account-a" + identity.CredentialScope = credentialScope(identity.UID, identity.DeviceID) + identity.ExpiredAt = fixedNow.Add(time.Hour).Unix() + mutated := false + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != exchangeGrantPath { + mutated = true + http.Error(writer, "unexpected", http.StatusInternalServerError) + return + } + http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) + writeEnvelope(writer, map[string]any{"uid": "account-b", "scope": loginGrantScope}) + })) + defer server.Close() + manager := NewManager(config.Load(), WithHTTPClient(server.Client()), withAuthBaseURLForTest(server.URL), withClockForTest(func() time.Time { return fixedNow })) + _, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ + Grant: "grant", RandomSecretKey: "secret", ExpireAt: fixedNow.Add(time.Minute).Unix(), + }, identity, LoginOptions{ForceRefresh: true, ExpectedCredentialScope: identity.CredentialScope}) + if !errors.Is(err, ErrCredentialAccountMismatch) || mutated { + t.Fatalf("error/mutated = %v/%v, want account mismatch before AK mutation", err, mutated) + } +} + +func TestForceRefreshNeverDeletesStoredTokenAfterAccountSwitch(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{6}, deviceIDBytes))) + identity.AccessKey = "account-a-ak" + identity.TokenID = "shared-looking-id" + identity.UID = "account-a" + identity.CredentialScope = credentialScope(identity.UID, identity.DeviceID) + identity.ExpiredAt = fixedNow.Add(time.Hour).Unix() + deleteCalled := false + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case exchangeGrantPath: + http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) + writeEnvelope(writer, map[string]any{"uid": "account-b", "scope": loginGrantScope}) + case queryAccessKeyPath: + writeEnvelope(writer, map[string]any{"access_token_list": []map[string]any{{ + "ak_id": identity.TokenID, "token": "account-b-ak", "expired_at": identity.ExpiredAt, + "token_name": identity.TokenName, "token_status": "enable", + }}}) + case deleteAccessKeyPath: + deleteCalled = true + http.Error(writer, "must not delete", http.StatusInternalServerError) + case generateAccessKeyPath: + writeEnvelope(writer, map[string]any{"ak": "account-b-replacement", "token_id": "account-b-id"}) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + manager := NewManager(config.Load(), WithHTTPClient(server.Client()), withAuthBaseURLForTest(server.URL), withClockForTest(func() time.Time { return fixedNow })) + credential, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ + Grant: "grant", RandomSecretKey: "secret", ExpireAt: fixedNow.Add(time.Minute).Unix(), + }, identity, LoginOptions{ForceRefresh: true}) + if err != nil || deleteCalled { + t.Fatalf("force refresh after account switch = %#v/%v, deleteCalled=%v", credentialWithoutSecret(credential), err, deleteCalled) + } + if credential.UID != "account-b" || credential.TokenID != "account-b-id" { + t.Fatalf("replacement credential = %#v", credentialWithoutSecret(credential)) + } +} + +func TestGenerateAccessKeyPermissionAndLimitGuidance(t *testing.T) { + for _, test := range []struct { + name string + ret string + message string + }{ + {name: "permission", ret: "3", message: "暂不具备创建 CLI Access Key 的权限"}, + {name: "possible limit", ret: "12001", message: "Access Key 数量上限"}, + } { + t.Run(test.name, func(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{5}, deviceIDBytes))) + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case exchangeGrantPath: + http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) + writeEnvelope(writer, map[string]any{"uid": "123", "scope": loginGrantScope}) + case queryAccessKeyPath: + writeEnvelope(writer, map[string]any{"access_token_list": []any{}}) + case generateAccessKeyPath: + _ = json.NewEncoder(writer).Encode(map[string]any{"ret": test.ret, "errmsg": "unsafe upstream detail"}) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + manager := NewManager(config.Load(), WithHTTPClient(server.Client()), withAuthBaseURLForTest(server.URL), withClockForTest(func() time.Time { return fixedNow })) + _, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{Grant: "grant", RandomSecretKey: "secret"}, identity, LoginOptions{}) + if err == nil || !strings.Contains(err.Error(), test.message) || strings.Contains(err.Error(), "unsafe upstream detail") { + t.Fatalf("error = %v, want safe guidance containing %q", err, test.message) + } + }) + } +} + +func TestCredentialScopeBindsUIDAndDeviceButNotAK(t *testing.T) { + device := base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{1}, deviceIDBytes)) + first := credentialScope("account-a", device) + if first != credentialScope("account-a", device) { + t.Fatal("scope changed for the same UID and device") + } + if first == credentialScope("account-b", device) { + t.Fatal("scope did not change across accounts") + } + otherDevice := base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{2}, deviceIDBytes)) + if first == credentialScope("account-a", otherDevice) { + t.Fatal("scope did not change across devices") + } +} + +func TestSelectManagedTokenRejectsAmbiguousExactNames(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{9}, deviceIDBytes))) + manager := NewManager(config.Load(), withClockForTest(func() time.Time { return fixedNow })) + tokens := []accessToken{ + {ID: "one", Token: "ak-one", Name: identity.TokenName, Status: "enable", ExpiredAt: fixedNow.Add(time.Hour).Unix()}, + {ID: "two", Token: "ak-two", Name: identity.TokenName, Status: "enable", ExpiredAt: fixedNow.Add(time.Hour).Unix()}, + } + if _, err := manager.selectManagedToken(tokens, identity); err == nil { + t.Fatal("ambiguous exact device tokens were accepted") + } + identity.TokenID = "two" + selected, err := manager.selectManagedToken(tokens, identity) + if err != nil { + t.Fatal(err) + } + if selected.ID != "two" { + t.Fatalf("selected ID = %q", selected.ID) + } + selected, err = manager.selectManagedToken([]accessToken{ + {ID: "two", Token: "renamed-ak", Name: "renamed-by-user", Status: "enable", ExpiredAt: fixedNow.Add(time.Hour).Unix()}, + }, identity) + if err != nil || selected == nil || selected.Token != "renamed-ak" { + t.Fatalf("renamed exact TokenID was not reused: %#v/%v", selected, err) + } +} + +func TestAuthErrorsDoNotLeakRequestOrResponseSecrets(t *testing.T) { + const ( + grant = "grant-must-stay-secret" + secret = "binding-must-stay-secret" + cookie = "cookie-must-stay-secret" + ) + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + http.SetCookie(writer, &http.Cookie{Name: "session", Value: cookie, Secure: true}) + writer.WriteHeader(http.StatusInternalServerError) + _, _ = writer.Write([]byte(grant + secret + cookie)) + })) + defer server.Close() + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{2}, deviceIDBytes))) + manager := NewManager(config.Load(), + WithCredentialStore(&memoryCredentialStore{}), + WithHTTPClient(server.Client()), + withAuthBaseURLForTest(server.URL), + ) + _, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ + Grant: grant, RandomSecretKey: secret, ExpireAt: time.Now().Add(time.Minute).Unix(), + }, identity, LoginOptions{}) + if err == nil { + t.Fatal("exchange error = nil") + } + for _, value := range []string{grant, secret, cookie} { + if strings.Contains(err.Error(), value) { + t.Fatalf("error leaked secret: %q", err) + } + } +} + +func TestResilientStoreFallsBackOnlyToSecureStore(t *testing.T) { + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{4}, deviceIDBytes))) + primary := &memoryCredentialStore{loadErr: ErrSecureStore, saveErr: ErrSecureStore, deleteErr: ErrSecureStore} + fallback := &memoryCredentialStore{} + store := &resilientCredentialStore{primary: primary, fallback: fallback} + if err := store.Save(context.Background(), identity); err != nil { + t.Fatal(err) + } + loaded, err := store.Load(context.Background()) + if err != nil || loaded.DeviceID != identity.DeviceID { + t.Fatalf("fallback load = %#v, %v", loaded, err) + } + if err := store.Delete(context.Background()); !errors.Is(err, ErrSecureStore) { + t.Fatalf("delete must report unresolved primary keyring failure, got %v", err) + } +} + +func TestResilientStoreDoesNotMaskCorruptPrimaryOrCleanupFailure(t *testing.T) { + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{4}, deviceIDBytes))) + corrupt := errors.New("corrupt primary credential") + primary := &memoryCredentialStore{loadErr: corrupt} + fallback := &memoryCredentialStore{credential: identity} + store := &resilientCredentialStore{primary: primary, fallback: fallback} + if _, err := store.Load(context.Background()); !errors.Is(err, corrupt) { + t.Fatalf("corrupt primary was masked by fallback: %v", err) + } + if fallback.loads != 0 { + t.Fatalf("fallback loads = %d, want zero after corrupt primary", fallback.loads) + } + + primary = &memoryCredentialStore{} + fallback = &memoryCredentialStore{credential: identity, deleteErr: ErrSecureStore} + store = &resilientCredentialStore{primary: primary, fallback: fallback} + if err := store.Save(context.Background(), identity); !errors.Is(err, ErrSecureStore) { + t.Fatalf("stale fallback cleanup error was swallowed: %v", err) + } + if primary.saves != 1 || fallback.deletes != 1 { + t.Fatalf("primary saves/fallback deletes = %d/%d", primary.saves, fallback.deletes) + } + + primary = &memoryCredentialStore{saveErr: errors.New("invalid primary write")} + fallback = &memoryCredentialStore{} + store = &resilientCredentialStore{primary: primary, fallback: fallback} + if err := store.Save(context.Background(), identity); err == nil || fallback.saves != 0 { + t.Fatalf("non-secure-store primary failure fell back: err=%v fallback saves=%d", err, fallback.saves) + } +} + +func TestManagerCachesCredentialAndLogoutPreservesDeviceIdentity(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + credential, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{3}, deviceIDBytes))) + credential.AccessKey = "stored-ak" + credential.TokenID = "stored-id" + credential.UID = "789" + credential.CredentialScope = credentialScope(credential.UID, credential.DeviceID) + credential.ExpiredAt = fixedNow.Add(time.Hour).Unix() + store := &memoryCredentialStore{credential: credential} + cfg := config.Load() + cfg.AccessKey = "" + manager := NewManager(cfg, WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) + + const callers = 32 + start := make(chan struct{}) + errs := make(chan error, callers) + var group sync.WaitGroup + for index := 0; index < callers; index++ { + group.Add(1) + go func() { + defer group.Done() + <-start + key, err := manager.ResolveAccessKey(context.Background()) + if err == nil && key != credential.AccessKey { + err = fmt.Errorf("key = %q", key) + } + errs <- err + }() + } + close(start) + group.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + if store.loads != 1 { + t.Fatalf("keyring loads = %d, want one process-local load", store.loads) + } + + replacement := cloneCredential(credential) + replacement.AccessKey = "replacement-ak" + replacement.TokenID = "replacement-id" + if err := manager.saveCredential(context.Background(), replacement); err != nil { + t.Fatal(err) + } + if key, err := manager.ResolveAccessKey(context.Background()); err != nil || key != replacement.AccessKey || store.loads != 1 { + t.Fatalf("cached replacement = %q/%v, loads=%d", key, err, store.loads) + } + + if err := manager.Logout(context.Background(), false); err != nil { + t.Fatal(err) + } + store.mu.Lock() + preserved := cloneCredential(store.credential) + deletes := store.deletes + store.mu.Unlock() + if preserved.DeviceID != credential.DeviceID || preserved.TokenName != credential.TokenName || + preserved.AccessKey != "" || preserved.TokenID != replacement.TokenID || preserved.UID != "" || preserved.CredentialScope != "" { + t.Fatalf("logout did not retain only reusable non-secret identity: %#v", credentialWithoutSecret(preserved)) + } + if deletes != 0 { + t.Fatalf("logout deleted the device identity %d time(s)", deletes) + } + if _, err := manager.ResolveAccessKey(context.Background()); !errors.Is(err, ErrCredentialNotFound) { + t.Fatalf("post-logout resolution = %v", err) + } + if store.loads != 2 { + t.Fatalf("post-logout cache was not cleared; loads=%d", store.loads) + } + selected, err := manager.selectManagedToken([]accessToken{{ + ID: replacement.TokenID, Token: replacement.AccessKey, Name: preserved.TokenName, + Status: "enable", ExpiredAt: replacement.ExpiredAt, + }}, preserved) + if err != nil || selected == nil || selected.ID != replacement.TokenID { + t.Fatalf("preserved identity could not reuse remote token: %#v/%v", selected, err) + } +} + +func TestManagerAuthOriginIgnoresRuntimeBaseURLAndPPE(t *testing.T) { + cfg := config.Load() + cfg.BaseURL = "https://untrusted.invalid" + cfg.PPEEnv = "ppe_untrusted" + manager := NewManager(cfg, WithCredentialStore(&memoryCredentialStore{})) + if got := manager.authBaseURL.String(); got != config.DefaultBaseURL { + t.Fatalf("auth base URL = %q, want %q", got, config.DefaultBaseURL) + } +} + +func TestResolveAccessKeyPrecedenceStatusAndScope(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{3}, deviceIDBytes))) + identity.AccessKey = "stored-ak" + identity.TokenID = "stored-id" + identity.UID = "789" + identity.CredentialScope = credentialScope(identity.UID, identity.DeviceID) + identity.ExpiredAt = fixedNow.Add(time.Hour).Unix() + store := &memoryCredentialStore{credential: identity} + cfg := config.Load() + cfg.AccessKey = " env-ak " + manager := NewManager(cfg, WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) + accessKey, err := manager.ResolveAccessKey(context.Background()) + if err != nil || accessKey != "env-ak" { + t.Fatalf("explicit resolution = %q, %v", accessKey, err) + } + status, err := manager.Status(context.Background()) + if err != nil || !status.LoggedIn || status.Source != "environment" { + t.Fatalf("environment status = %#v, %v", status, err) + } + + cfg.AccessKey = "" + accessKey, err = manager.ResolveAccessKey(context.Background()) + if err != nil || accessKey != "stored-ak" { + t.Fatalf("stored resolution = %q, %v", accessKey, err) + } + scope, err := manager.CredentialScope(context.Background()) + if err != nil || scope != identity.CredentialScope { + t.Fatalf("scope = %q, %v", scope, err) + } + expired := cloneCredential(identity) + expired.ExpiredAt = fixedNow.Unix() + if err := manager.saveCredential(context.Background(), expired); err != nil { + t.Fatal(err) + } + if _, err := manager.ResolveAccessKey(context.Background()); !errors.Is(err, ErrCredentialExpired) { + t.Fatalf("expired error = %v", err) + } + if err := manager.Logout(context.Background(), true); !errors.Is(err, ErrRemoteRevokeUnsupported) { + t.Fatalf("revoke error = %v", err) + } + if err := manager.Logout(context.Background(), false); err != nil { + t.Fatal(err) + } +} + +func TestFileCredentialStoreIsPrivateAtomicAndNoFollow(t *testing.T) { + directory := filepath.Join(t.TempDir(), "auth") + path := filepath.Join(directory, credentialFileName) + store := NewFileCredentialStore(path) + credential, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{5}, deviceIDBytes))) + credential.AccessKey = "file-ak" + credential.TokenID = "file-id" + credential.UID = "100" + credential.CredentialScope = credentialScope(credential.UID, credential.DeviceID) + credential.ExpiredAt = time.Now().Add(time.Hour).Unix() + if err := store.Save(context.Background(), credential); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("credential mode = %o", info.Mode().Perm()) + } + loaded, err := store.Load(context.Background()) + if err != nil || loaded.AccessKey != credential.AccessKey { + t.Fatalf("loaded credential = %#v, %v", credentialWithoutSecret(loaded), err) + } + + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + target := filepath.Join(t.TempDir(), "target") + if err := os.WriteFile(target, []byte("must-not-change"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, path); err != nil { + t.Fatal(err) + } + if _, err := store.Load(context.Background()); !errors.Is(err, ErrSecureStore) { + t.Fatalf("symlink load error = %v", err) + } + if err := store.Save(context.Background(), credential); err != nil { + t.Fatal(err) + } + targetData, _ := os.ReadFile(target) + if string(targetData) != "must-not-change" { + t.Fatal("atomic save followed and overwrote symlink target") + } +} + +func TestSanitizedBrowserEnv(t *testing.T) { + input := []string{ + "PATH=/usr/bin", + "XYQ_ACCESS_KEY=secret", + "PIPPIT_TOKEN=secret", + "PIPPIT_CLI_AK=secret", + "PIPPIT_CLI_PPE_ENV=ppe_safe", + "OTHER_TOKEN=unrelated", + } + got := SanitizedBrowserEnv(input) + joined := strings.Join(got, "\n") + for _, forbidden := range []string{"XYQ_ACCESS_KEY", "PIPPIT_TOKEN", "PIPPIT_CLI_AK"} { + if strings.Contains(joined, forbidden) { + t.Fatalf("browser env retained %s", forbidden) + } + } + for _, wanted := range []string{"PATH=/usr/bin", "PIPPIT_CLI_PPE_ENV=ppe_safe", "OTHER_TOKEN=unrelated"} { + if !strings.Contains(joined, wanted) { + t.Fatalf("browser env removed %s", wanted) + } + } +} + +func postCallback(t *testing.T, callbackURL, origin string, payload loginGrantPayload) int { + t.Helper() + status, err := sendCallback(callbackURL, origin, payload) + if err != nil { + t.Fatal(err) + } + return status +} + +func sendCallback(callbackURL, origin string, payload loginGrantPayload) (int, error) { + body, err := json.Marshal(payload) + if err != nil { + return 0, err + } + request, err := http.NewRequest(http.MethodPost, callbackURL, bytes.NewReader(body)) + if err != nil { + return 0, err + } + request.Header.Set("Origin", origin) + request.Header.Set("Content-Type", "application/json") + response, err := http.DefaultClient.Do(request) + if err != nil { + return 0, err + } + _, _ = io.Copy(io.Discard, response.Body) + response.Body.Close() + return response.StatusCode, nil +} + +func writeEnvelope(writer http.ResponseWriter, data any) { + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]any{"ret": "0", "data": data}) +} + +func assertAuthHeaders(t *testing.T, request *http.Request) { + t.Helper() + want := map[string]string{"appvr": "1.1.4", "entrance-from": "web", "appid": "795647"} + for name, value := range want { + if got := request.Header.Get(name); got != value { + t.Errorf("header %s = %q, want %q", name, got, value) + } + } +} + +func credentialWithoutSecret(credential *Credential) any { + if credential == nil { + return nil + } + return struct { + DeviceID string + CredentialScope string + TokenName string + TokenID string + UID string + ExpiredAt int64 + }{credential.DeviceID, credential.CredentialScope, credential.TokenName, credential.TokenID, credential.UID, credential.ExpiredAt} +} From ee10a44d64c77a6fa1ebe2b181e17efdf5080d5c Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 07:20:20 +0800 Subject: [PATCH 40/48] feat(cli): add browser login commands Co-authored-by: Codex <codex@openai.com> --- cmd/auth/auth.go | 278 ++++++++++++++--------------- cmd/auth/auth_test.go | 125 +++++++++++++ cmd/root.go | 19 +- cmd/root_test.go | 11 ++ cmd/short_drama_test.go | 9 +- internal/common/access_key.go | 27 ++- internal/common/access_key_test.go | 38 +++- internal/common/runner.go | 15 ++ internal/config/config.go | 20 --- internal/config/config_test.go | 18 -- 10 files changed, 363 insertions(+), 197 deletions(-) create mode 100644 cmd/auth/auth_test.go diff --git a/cmd/auth/auth.go b/cmd/auth/auth.go index 6cdc1c5..5d6883f 100644 --- a/cmd/auth/auth.go +++ b/cmd/auth/auth.go @@ -1,144 +1,138 @@ package authcmd -//import ( -// "fmt" -// "io" -// "strings" -// "time" -// -// "github.com/Pippit-dev/pippit-cli/internal/auth" -// "github.com/Pippit-dev/pippit-cli/internal/common" -// "github.com/bytedance/sonic" -// "github.com/spf13/cobra" -//) -// -//type checkResult struct { -// Pending bool `json:"pending"` -// State any `json:"state,omitempty"` -//} -// -//func NewCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { -// cmd := &cobra.Command{ -// Use: "auth", -// Short: "Manage Pippit OAuth login state", -// } -// cmd.SetOut(stdout) -// cmd.SetErr(stderr) -// cmd.AddCommand(newLoginCommand(stdout, stderr, runner)) -// cmd.AddCommand(newCheckCommand(stdout, stderr, runner)) -// cmd.AddCommand(newStatusCommand(stdout, stderr, runner)) -// cmd.AddCommand(newLogoutCommand(stdout, stderr, runner)) -// return cmd -//} -// -//func newLoginCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { -// cmd := &cobra.Command{ -// Use: "login", -// Short: "Start an OAuth device login flow", -// Args: cobra.NoArgs, -// RunE: func(cmd *cobra.Command, _ []string) error { -// if runner == nil || runner.AuthAuthorizer == nil { -// return fmt.Errorf("auth manager is required") -// } -// flow, err := runner.AuthAuthorizer.NewLoginFlow(cmd.Context()) -// if err != nil { -// return err -// } -// return writeJSON(stdout, flow) -// }, -// } -// cmd.SetOut(stdout) -// cmd.SetErr(stderr) -// return cmd -//} -// -//func newCheckCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { -// var deviceCode string -// cmd := &cobra.Command{ -// Use: "check", -// Short: "Check whether an OAuth device login has completed", -// Args: cobra.NoArgs, -// RunE: func(cmd *cobra.Command, _ []string) error { -// deviceCode = strings.TrimSpace(deviceCode) -// if deviceCode == "" { -// return fmt.Errorf("--device-code is required") -// } -// if runner == nil || runner.AuthAuthorizer == nil { -// return fmt.Errorf("auth manager is required") -// } -// state, err := runner.AuthAuthorizer.CheckLogin(cmd.Context(), deviceCode) -// if auth.IsLoginPending(err) { -// return writeJSON(stdout, checkResult{Pending: true}) -// } -// if err != nil { -// return err -// } -// v := map[string]any{ -// "logged_in": state.LoggedIn, -// "expires_at": state.ExpiresAt.Format(time.RFC3339), -// } -// return writeJSON(stdout, checkResult{State: v}) -// }, -// } -// cmd.SetOut(stdout) -// cmd.SetErr(stderr) -// cmd.Flags().StringVar(&deviceCode, "device-code", "", "device code returned by auth login") -// return cmd -//} -// -//func newStatusCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { -// cmd := &cobra.Command{ -// Use: "status", -// Short: "Show current OAuth login state", -// Args: cobra.NoArgs, -// RunE: func(cmd *cobra.Command, _ []string) error { -// if runner == nil || runner.AuthAuthorizer == nil { -// return fmt.Errorf("auth manager is required") -// } -// state, err := runner.AuthAuthorizer.State(cmd.Context()) -// if err != nil { -// return err -// } -// if !state.LoggedIn { -// return fmt.Errorf("not logged in") -// } -// v := map[string]any{ -// "logged_in": state.LoggedIn, -// "expires_at": state.ExpiresAt.Format(time.RFC3339), -// } -// return writeJSON(stdout, v) -// }, -// } -// cmd.SetOut(stdout) -// cmd.SetErr(stderr) -// return cmd -//} -// -//func newLogoutCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { -// cmd := &cobra.Command{ -// Use: "logout", -// Short: "Clear local OAuth login state", -// Args: cobra.NoArgs, -// RunE: func(cmd *cobra.Command, _ []string) error { -// if runner == nil || runner.AuthAuthorizer == nil { -// return fmt.Errorf("auth manager is required") -// } -// if err := runner.AuthAuthorizer.Logout(cmd.Context()); err != nil { -// return err -// } -// return writeJSON(stdout, map[string]bool{"logged_out": true}) -// }, -// } -// cmd.SetOut(stdout) -// cmd.SetErr(stderr) -// return cmd -//} -// -//func writeJSON(w io.Writer, v any) error { -// data, err := sonic.Marshal(v) -// if err != nil { -// return err -// } -// _, err = fmt.Fprintln(w, string(data)) -// return err -//} +import ( + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" + + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" + "github.com/Pippit-dev/pippit-cli/internal/common" + "github.com/spf13/cobra" +) + +type loginResult struct { + LoggedIn bool `json:"logged_in"` + Source string `json:"source"` + UID string `json:"uid,omitempty"` + CredentialScope string `json:"credential_scope,omitempty"` + ExpiresAt string `json:"expires_at,omitempty"` +} + +type logoutResult struct { + LoggedOut bool `json:"logged_out"` + EnvironmentStillActive bool `json:"environment_still_active,omitempty"` + RemoteCredentialPreserved bool `json:"remote_credential_preserved"` +} + +// NewLoginCommand creates the top-level `pippit-tool-cli login` command. +func NewLoginCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { + var forceRefresh bool + command := &cobra.Command{ + Use: "login", + Short: "通过浏览器登录小云雀 CLI", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + manager, err := requireAuthManager(runner) + if err != nil { + return err + } + credential, err := manager.Login(command.Context(), internal_auth.LoginOptions{ + Progress: stderr, ForceRefresh: forceRefresh, + }) + if err != nil { + return err + } + if runner.Config != nil && strings.TrimSpace(runner.Config.AccessKey) != "" { + _, _ = fmt.Fprintln(stderr, "提示：当前进程设置了 XYQ_ACCESS_KEY，它会继续优先于刚保存的浏览器登录凭证。") + } + return writeJSON(stdout, loginResult{ + LoggedIn: true, + Source: "browser", + UID: credential.UID, + CredentialScope: credential.CredentialScope, + ExpiresAt: time.Unix(credential.ExpiredAt, 0).Format(time.RFC3339), + }) + }, + } + command.SetOut(stdout) + command.SetErr(stderr) + command.Flags().BoolVar(&forceRefresh, "force", false, "强制轮换当前设备的 CLI Access Key（仅在旧密钥被拒绝时使用）") + return command +} + +// NewStatusCommand creates the top-level `pippit-tool-cli status` command. +func NewStatusCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { + command := &cobra.Command{ + Use: "status", + Short: "查看小云雀 CLI 登录状态", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + manager, err := requireAuthManager(runner) + if err != nil { + return err + } + status, err := manager.Status(command.Context()) + if err != nil { + return err + } + result := loginResult{LoggedIn: status.LoggedIn, Source: status.Source, UID: status.UID, CredentialScope: status.CredentialScope} + if !status.ExpiresAt.IsZero() { + result.ExpiresAt = status.ExpiresAt.Format(time.RFC3339) + } + return writeJSON(stdout, result) + }, + } + command.SetOut(stdout) + command.SetErr(stderr) + return command +} + +// NewLogoutCommand clears only the browser-managed local credential. An +// explicit XYQ_ACCESS_KEY belongs to the caller's environment and is never +// modified or revoked by this command. +func NewLogoutCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Command { + command := &cobra.Command{ + Use: "logout", + Short: "清除本机小云雀 CLI 登录（不撤销远程 Access Key）", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + manager, err := requireAuthManager(runner) + if err != nil { + return err + } + if err := manager.Logout(command.Context(), false); err != nil && !errors.Is(err, internal_auth.ErrCredentialNotFound) { + return err + } + environmentActive := runner.Config != nil && strings.TrimSpace(runner.Config.AccessKey) != "" + if environmentActive { + _, _ = fmt.Fprintln(stderr, "提示：XYQ_ACCESS_KEY 仍由当前 shell 提供，CLI 无法替你清除该环境变量。") + } + _, _ = fmt.Fprintln(stderr, "已清除本机登录密钥；远程 Access Key 未撤销，并保留非秘密设备标识供下次登录安全复用。") + return writeJSON(stdout, logoutResult{ + LoggedOut: true, EnvironmentStillActive: environmentActive, RemoteCredentialPreserved: true, + }) + }, + } + command.SetOut(stdout) + command.SetErr(stderr) + return command +} + +func requireAuthManager(runner *common.Runner) (common.AuthManager, error) { + if runner == nil || runner.Auth == nil { + return nil, fmt.Errorf("小云雀 CLI 浏览器授权尚未配置") + } + return runner.Auth, nil +} + +func writeJSON(writer io.Writer, value any) error { + payload, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("编码登录结果失败: %w", err) + } + _, err = fmt.Fprintln(writer, string(payload)) + return err +} diff --git a/cmd/auth/auth_test.go b/cmd/auth/auth_test.go new file mode 100644 index 0000000..65f8c94 --- /dev/null +++ b/cmd/auth/auth_test.go @@ -0,0 +1,125 @@ +package authcmd + +import ( + "bytes" + "context" + "encoding/json" + "io" + "strings" + "testing" + "time" + + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" + "github.com/Pippit-dev/pippit-cli/internal/common" + "github.com/Pippit-dev/pippit-cli/internal/config" +) + +type fakeAuthManager struct { + credential *internal_auth.Credential + status *internal_auth.Status + loginCalls int + loginOptions []internal_auth.LoginOptions + logoutCalls int +} + +func (manager *fakeAuthManager) ResolveAccessKey(context.Context) (string, error) { + if manager.credential == nil { + return "", internal_auth.ErrCredentialNotFound + } + return manager.credential.AccessKey, nil +} + +func (manager *fakeAuthManager) Login(_ context.Context, options internal_auth.LoginOptions) (*internal_auth.Credential, error) { + manager.loginCalls++ + manager.loginOptions = append(manager.loginOptions, options) + if options.Progress != nil { + _, _ = options.Progress.Write([]byte("正在完成浏览器授权…\n")) + } + return manager.credential, nil +} + +func TestLoginCommandCanRequestSafeCredentialRotation(t *testing.T) { + manager := &fakeAuthManager{credential: &internal_auth.Credential{ + UID: "123", CredentialScope: "account-device-scope", ExpiredAt: time.Now().Add(time.Hour).Unix(), + }} + command := NewLoginCommand(io.Discard, io.Discard, &common.Runner{Config: &config.Config{}, Auth: manager}) + command.SetArgs([]string{"--force"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if len(manager.loginOptions) != 1 || !manager.loginOptions[0].ForceRefresh { + t.Fatalf("login options = %#v, want forced refresh", manager.loginOptions) + } +} + +func (manager *fakeAuthManager) Status(context.Context) (*internal_auth.Status, error) { + return manager.status, nil +} + +func (manager *fakeAuthManager) Logout(context.Context, bool) error { + manager.logoutCalls++ + return nil +} + +func (manager *fakeAuthManager) CredentialScope(context.Context) (string, error) { + return "device-scope", nil +} + +func TestLoginCommandPrintsMetadataWithoutAccessKey(t *testing.T) { + const accessKey = "must-never-be-printed" + expiresAt := time.Now().Add(time.Hour).Unix() + manager := &fakeAuthManager{credential: &internal_auth.Credential{ + AccessKey: accessKey, UID: "123", CredentialScope: "device-scope", ExpiredAt: expiresAt, + }} + var stdout, stderr bytes.Buffer + command := NewLoginCommand(&stdout, &stderr, &common.Runner{Config: &config.Config{}, Auth: manager}) + command.SetArgs(nil) + if err := command.Execute(); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if manager.loginCalls != 1 || !strings.Contains(stderr.String(), "浏览器授权") { + t.Fatalf("login calls/stderr = %d/%q", manager.loginCalls, stderr.String()) + } + if strings.Contains(stdout.String(), accessKey) || strings.Contains(stderr.String(), accessKey) { + t.Fatalf("command output leaked Access Key: stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + var result loginResult + if err := json.Unmarshal(stdout.Bytes(), &result); err != nil { + t.Fatal(err) + } + if !result.LoggedIn || result.Source != "browser" || result.UID != "123" || result.CredentialScope != "device-scope" { + t.Fatalf("login result = %#v", result) + } +} + +func TestStatusAndLogoutDoNotModifyExplicitEnvironmentCredential(t *testing.T) { + manager := &fakeAuthManager{status: &internal_auth.Status{LoggedIn: true, Source: "environment"}} + runner := &common.Runner{Config: &config.Config{AccessKey: "explicit-ci-key"}, Auth: manager} + var statusOut, statusErr bytes.Buffer + status := NewStatusCommand(&statusOut, &statusErr, runner) + if err := status.Execute(); err != nil { + t.Fatal(err) + } + if strings.Contains(statusOut.String(), runner.Config.AccessKey) { + t.Fatal("status output leaked explicit Access Key") + } + + var logoutOut, logoutErr bytes.Buffer + logout := NewLogoutCommand(&logoutOut, &logoutErr, runner) + if err := logout.Execute(); err != nil { + t.Fatal(err) + } + if manager.logoutCalls != 1 || runner.Config.AccessKey != "explicit-ci-key" { + t.Fatalf("logout calls/key = %d/%q", manager.logoutCalls, runner.Config.AccessKey) + } + if !strings.Contains(logoutErr.String(), "无法替你清除") || strings.Contains(logoutErr.String(), runner.Config.AccessKey) { + t.Fatalf("logout stderr = %q", logoutErr.String()) + } + var result logoutResult + if err := json.Unmarshal(logoutOut.Bytes(), &result); err != nil { + t.Fatal(err) + } + if !result.LoggedOut || !result.RemoteCredentialPreserved || !strings.Contains(logoutErr.String(), "远程 Access Key 未撤销") { + t.Fatalf("logout result/stderr = %#v/%q", result, logoutErr.String()) + } +} diff --git a/cmd/root.go b/cmd/root.go index 47365cb..6c7e17f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,18 +1,20 @@ package cmd import ( + "context" "fmt" "io" "os" "strings" - // authcmd "github.com/Pippit-dev/pippit-cli/cmd/auth" + authcmd "github.com/Pippit-dev/pippit-cli/cmd/auth" canvascmd "github.com/Pippit-dev/pippit-cli/cmd/canvas" "github.com/Pippit-dev/pippit-cli/cmd/generate_image" "github.com/Pippit-dev/pippit-cli/cmd/generate_video" "github.com/Pippit-dev/pippit-cli/cmd/short_drama" updatecmd "github.com/Pippit-dev/pippit-cli/cmd/update" "github.com/Pippit-dev/pippit-cli/cmd/video_tool" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/common" "github.com/Pippit-dev/pippit-cli/internal/config" "github.com/Pippit-dev/pippit-cli/internal/version" @@ -32,14 +34,15 @@ func NewRootCommand(stdout, stderr io.Writer) *cobra.Command { func newRootRunner(cfg *config.Config) *common.Runner { runner := common.NewRunner(cfg, nil) + runner.Auth = internal_auth.NewManager(cfg) runner.Client = common.NewHTTPClientWithPPEEnv( cfg.BaseURL, cfg.HTTPTimeout, - common.NewAccessKeyProviderAuthorizer(func() string { - if runner.Config == nil { - return "" + common.NewAccessKeyContextProviderAuthorizer(func(ctx context.Context) (string, error) { + if runner.Auth == nil { + return "", nil } - return runner.Config.AccessKey + return runner.Auth.ResolveAccessKey(ctx) }), func() string { return cfg.PPEEnv }, ) @@ -60,7 +63,9 @@ func newRootCommand(stdout, stderr io.Writer, runner *common.Runner) *cobra.Comm root.SetOut(stdout) root.SetErr(stderr) configurePPEFlag(root, runner.Config) - // root.AddCommand(authcmd.NewCommand(stdout, stderr, runner)) // temporarily disabled; auth is via access key injection + root.AddCommand(authcmd.NewLoginCommand(stdout, stderr, runner)) + root.AddCommand(authcmd.NewStatusCommand(stdout, stderr, runner)) + root.AddCommand(authcmd.NewLogoutCommand(stdout, stderr, runner)) root.AddCommand(canvascmd.NewCommand(stdout, stderr, runner)) root.AddCommand(newDownloadResultCommand(stdout, stderr, runner)) root.AddCommand(newGetThreadCommand(stdout, stderr, runner)) @@ -81,7 +86,7 @@ func configurePPEFlag(root *cobra.Command, cfg *config.Config) { &cfg.PPEEnv, "ppe-env", cfg.PPEEnv, - "route Pippit API requests to a PPE environment (for example, ppe_cli_canvas_ak)", + "将小云雀业务 API 路由到 PPE（例如 ppe_cli_canvas_ak；网页登录始终使用生产身份域）", ) root.PersistentPreRunE = func(_ *cobra.Command, _ []string) error { ppeEnv, err := config.NormalizePPEEnv(cfg.PPEEnv) diff --git a/cmd/root_test.go b/cmd/root_test.go index 4ba96bc..0a4d220 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -115,6 +115,17 @@ func TestRootHelpIncludesPPEFlag(t *testing.T) { } } +func TestRootRegistersTopLevelBrowserAuthCommands(t *testing.T) { + var stdout, stderr bytes.Buffer + root := NewRootCommand(&stdout, &stderr) + for _, name := range []string{"login", "status", "logout"} { + command, _, err := root.Find([]string{name}) + if err != nil || command == nil || command.Name() != name { + t.Fatalf("root.Find(%q) = %#v, %v", name, command, err) + } + } +} + func newPPEFlagTestRoot(t *testing.T) (*config.Config, *cobra.Command, *bool) { t.Helper() cfg := config.Load() diff --git a/cmd/short_drama_test.go b/cmd/short_drama_test.go index b6706a0..cd59040 100644 --- a/cmd/short_drama_test.go +++ b/cmd/short_drama_test.go @@ -971,14 +971,11 @@ func assertAccessKeyGuidance(t *testing.T, err error) { t.Fatal("error = nil, want access key guidance") } msg := err.Error() - if !strings.Contains(msg, "XYQ_ACCESS_KEY 缺失") { + if !strings.Contains(msg, "pippit-tool-cli login") { t.Fatalf("error = %q, want access key guidance", err) } - if !strings.Contains(msg, "https://xyq.jianying.com/home?tab_name=home") { - t.Fatalf("error = %q, want access key settings URL", err) - } - if !strings.Contains(msg, `export XYQ_ACCESS_KEY="<your-access-key>"`) { - t.Fatalf("error = %q, want setup command guidance", err) + if !strings.Contains(msg, "XYQ_ACCESS_KEY") { + t.Fatalf("error = %q, want CI override guidance", err) } } diff --git a/internal/common/access_key.go b/internal/common/access_key.go index 042693a..a0ab704 100644 --- a/internal/common/access_key.go +++ b/internal/common/access_key.go @@ -2,15 +2,17 @@ package common import ( "context" + "errors" "fmt" "net/http" "strings" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/config" ) type accessKeyAuthorizer struct { - accessKey func() string + accessKey func(context.Context) (string, error) } func NewAccessKeyAuthorizer(accessKey string) RequestAuthorizer { @@ -25,6 +27,18 @@ func NewAccessKeyProviderAuthorizer(accessKey func() string) RequestAuthorizer { if accessKey == nil { accessKey = func() string { return "" } } + return NewAccessKeyContextProviderAuthorizer(func(context.Context) (string, error) { + return accessKey(), nil + }) +} + +// NewAccessKeyContextProviderAuthorizer resolves a credential lazily for each +// request. This keeps keychain access out of --help and lets a browser login in +// the same process authorize the next request without rebuilding the client. +func NewAccessKeyContextProviderAuthorizer(accessKey func(context.Context) (string, error)) RequestAuthorizer { + if accessKey == nil { + accessKey = func(context.Context) (string, error) { return "", nil } + } return &accessKeyAuthorizer{accessKey: accessKey} } @@ -34,10 +48,17 @@ func (a *accessKeyAuthorizer) Inject(ctx context.Context, req *http.Request) err } accessKey := "" if a != nil && a.accessKey != nil { - accessKey = strings.TrimSpace(a.accessKey()) + resolved, err := a.accessKey(ctx) + if err != nil && req.Method == http.MethodPost { + if errors.Is(err, internal_auth.ErrCredentialNotFound) || errors.Is(err, internal_auth.ErrCredentialExpired) { + return fmt.Errorf("未找到可用的小云雀登录凭证；请先运行 pippit-tool-cli login: %w", err) + } + return fmt.Errorf("读取小云雀 CLI 安全登录凭证失败；请检查本机凭证库后重试: %w", err) + } + accessKey = strings.TrimSpace(resolved) } if accessKey == "" && req.Method == http.MethodPost { - return fmt.Errorf("%s 缺失；请前往小云雀官网个人设置页创建 Access Key，地址：https://xyq.jianying.com/home?tab_name=home\n配置后重试：\n export %s=\"<your-access-key>\"", config.EnvXYQAccessKey, config.EnvXYQAccessKey) + return fmt.Errorf("未登录小云雀 CLI；请先运行 pippit-tool-cli login（CI 仍可设置 %s）", config.EnvXYQAccessKey) } if accessKey != "" { req.Header.Set("Authorization", "Bearer "+accessKey) diff --git a/internal/common/access_key_test.go b/internal/common/access_key_test.go index 24a94b5..30b0812 100644 --- a/internal/common/access_key_test.go +++ b/internal/common/access_key_test.go @@ -2,9 +2,12 @@ package common import ( "context" + "errors" "net/http" "strings" "testing" + + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" ) func TestAccessKeyProviderAuthorizerReadsLatestValue(t *testing.T) { @@ -35,7 +38,7 @@ func TestAccessKeyProviderAuthorizerRejectsMissingKeyWithoutLeakingPriorValue(t accessKey = " " err := authorizer.Inject(context.Background(), newAccessKeyTestRequest(t, http.MethodPost)) - if err == nil || !strings.Contains(err.Error(), "XYQ_ACCESS_KEY 缺失") { + if err == nil || !strings.Contains(err.Error(), "pippit-tool-cli login") { t.Fatalf("Inject() error = %v, want missing Access Key guidance", err) } if strings.Contains(err.Error(), "prior-secret") { @@ -43,6 +46,39 @@ func TestAccessKeyProviderAuthorizerRejectsMissingKeyWithoutLeakingPriorValue(t } } +func TestAccessKeyContextProviderAuthorizerLoadsSavedCredentialLazily(t *testing.T) { + called := 0 + authorizer := NewAccessKeyContextProviderAuthorizer(func(ctx context.Context) (string, error) { + called++ + if err := ctx.Err(); err != nil { + return "", err + } + return " saved-browser-key ", nil + }) + request := newAccessKeyTestRequest(t, http.MethodPost) + if called != 0 { + t.Fatal("credential provider was called before request authorization") + } + if err := authorizer.Inject(context.Background(), request); err != nil { + t.Fatalf("Inject() error = %v", err) + } + if called != 1 || request.Header.Get("Authorization") != "Bearer saved-browser-key" { + t.Fatalf("called=%d Authorization=%q", called, request.Header.Get("Authorization")) + } +} + +func TestAccessKeyContextProviderAuthorizerPreservesCredentialStateErrors(t *testing.T) { + for _, cause := range []error{internal_auth.ErrCredentialNotFound, internal_auth.ErrCredentialExpired} { + authorizer := NewAccessKeyContextProviderAuthorizer(func(context.Context) (string, error) { + return "", cause + }) + err := authorizer.Inject(context.Background(), newAccessKeyTestRequest(t, http.MethodPost)) + if !errors.Is(err, cause) || !strings.Contains(err.Error(), "pippit-tool-cli login") { + t.Fatalf("Inject() error = %v, want wrapped %v", err, cause) + } + } +} + func TestAccessKeyProviderAuthorizerAllowsUnauthenticatedRead(t *testing.T) { authorizer := NewAccessKeyProviderAuthorizer(nil) request := newAccessKeyTestRequest(t, http.MethodGet) diff --git a/internal/common/runner.go b/internal/common/runner.go index 29e567d..5f9f411 100644 --- a/internal/common/runner.go +++ b/internal/common/runner.go @@ -1,13 +1,28 @@ package common import ( + "context" + + "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/config" ) +// AuthManager is the credential boundary shared by native CLI commands. It +// resolves explicit CI credentials first and browser-managed credentials +// second, without exposing either value to command output. +type AuthManager interface { + ResolveAccessKey(context.Context) (string, error) + Login(context.Context, auth.LoginOptions) (*auth.Credential, error) + Status(context.Context) (*auth.Status, error) + Logout(context.Context, bool) error + CredentialScope(context.Context) (string, error) +} + // Runner carries runtime dependencies for command execution. type Runner struct { Config *config.Config Client Client + Auth AuthManager } func NewRunner(cfg *config.Config, client Client) *Runner { diff --git a/internal/config/config.go b/internal/config/config.go index 2d3371c..77a81ff 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,8 +12,6 @@ const ( DefaultBaseURL = "https://xyq.jianying.com" DefaultHTTPTimeout = 30 * time.Minute DefaultAuthTTL = 30 * time.Second - DefaultOAuthClientKey = "mock-cli" - DefaultOAuthBaseURL = "https://passport.bytedance.com" DefaultAuthStoreServiceName = "pippit-cli" SubmitRunPath = "/api/biz/v1/skill/submit_run" GetThreadPath = "/api/biz/v1/skill/get_thread" @@ -33,17 +31,9 @@ type Config struct { AuthTTL time.Duration AccessKey string PPEEnv string - OAuth *OAuth Paths *Paths } -type OAuth struct { - ClientKey string - BaseURL string - StoreServiceName string - Scopes []string -} - type Paths struct { SubmitRun string GetThread string @@ -59,7 +49,6 @@ func Load() *Config { AuthTTL: DefaultAuthTTL, AccessKey: strings.TrimSpace(os.Getenv(EnvXYQAccessKey)), PPEEnv: strings.TrimSpace(os.Getenv(EnvPPEEnv)), - OAuth: resolveOAuth(), Paths: &Paths{ SubmitRun: SubmitRunPath, GetThread: GetThreadPath, @@ -82,12 +71,3 @@ func NormalizePPEEnv(value string) (string, error) { } return value, nil } - -func resolveOAuth() *OAuth { - return &OAuth{ - ClientKey: DefaultOAuthClientKey, - BaseURL: DefaultOAuthBaseURL, - StoreServiceName: DefaultAuthStoreServiceName, - Scopes: []string{"user_info", "aigc_generate"}, - } -} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 561e1af..01e88c4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -21,24 +21,6 @@ func TestLoadUsesDefaultConfig(t *testing.T) { if cfg.PPEEnv != "" { t.Fatalf("PPEEnv = %q, want empty", cfg.PPEEnv) } - if cfg.OAuth.ClientKey != DefaultOAuthClientKey { - t.Fatalf("OAuth.ClientKey = %q, want %q", cfg.OAuth.ClientKey, DefaultOAuthClientKey) - } - if cfg.OAuth.StoreServiceName != DefaultAuthStoreServiceName { - t.Fatalf("OAuth.StoreServiceName = %q, want %q", cfg.OAuth.StoreServiceName, DefaultAuthStoreServiceName) - } - if cfg.OAuth.BaseURL != DefaultOAuthBaseURL { - t.Fatalf("OAuth.BaseURL = %q, want %q", cfg.OAuth.BaseURL, DefaultOAuthBaseURL) - } - wantScopes := []string{"user_info", "aigc_generate"} - if len(cfg.OAuth.Scopes) != len(wantScopes) { - t.Fatalf("OAuth.Scopes = %#v, want %#v", cfg.OAuth.Scopes, wantScopes) - } - for i := range wantScopes { - if cfg.OAuth.Scopes[i] != wantScopes[i] { - t.Fatalf("OAuth.Scopes = %#v, want %#v", cfg.OAuth.Scopes, wantScopes) - } - } if cfg.Paths.SubmitRun != SubmitRunPath { t.Fatalf("SubmitRun path = %q, want %q", cfg.Paths.SubmitRun, SubmitRunPath) } From 4fb7e02b8a39fee6ba478ead3c54912002e5bd68 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 07:23:48 +0800 Subject: [PATCH 41/48] feat(canvas): resume imports through browser reauthorization Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import.go | 55 ++++--- cmd/canvas/import_auth.go | 197 +++++++++++++++++-------- cmd/canvas/import_auth_flow.go | 47 +++--- cmd/canvas/import_auth_flow_test.go | 168 +++++++++++++++++---- cmd/canvas/import_auth_test.go | 149 ++++++++++++------- cmd/canvas/import_media.go | 16 +- cmd/canvas/import_prompt.go | 19 --- cmd/canvas/import_prompt_tui.go | 20 --- cmd/canvas/import_prompt_tui_test.go | 24 --- cmd/canvas/import_test.go | 16 +- internal/canvas/canvas_test.go | 56 +++++++ internal/canvas/create.go | 21 +++ internal/canvas/import_facade.go | 3 + internal/canvasplan/canvasplan_test.go | 105 +++++++++++++ internal/canvasplan/executor.go | 19 +++ 15 files changed, 646 insertions(+), 269 deletions(-) diff --git a/cmd/canvas/import.go b/cmd/canvas/import.go index 6206be8..79350b9 100644 --- a/cmd/canvas/import.go +++ b/cmd/canvas/import.go @@ -60,7 +60,7 @@ type importDependencies struct { userCacheDir func() (string, error) userConfigDir func() (string, error) target func() string - authScope func() string + authScope func(context.Context) (string, error) isInteractive func(io.Reader) bool mediaPoll time.Duration mediaTimeout time.Duration @@ -100,7 +100,9 @@ func newImportDependencies(runner *common.Runner) importDependencies { userCacheDir: os.UserCacheDir, userConfigDir: os.UserConfigDir, target: func() string { return canvasImportTarget(runner) }, - authScope: func() string { return canvasImportAuthScope(runner) }, + authScope: func(ctx context.Context) (string, error) { + return runnerImportAuthAPI{runner: runner}.CredentialScope(ctx) + }, isInteractive: importInputIsInteractive, mediaPoll: defaultImportMediaPollInterval, mediaTimeout: defaultImportMediaWaitTimeout, @@ -182,6 +184,19 @@ func runCanvasImport( if err := preflightCanvasImportAuth(ctx, dependencies, prompts, stderr); err != nil { return nil, err } + // Bind the whole durable import to the authenticated UID and this device + // before exporting or creating checkpoints. Any mid-run browser login must + // return to this exact scope or the operation stops without reusing state. + authScope := "" + if dependencies.authScope != nil { + authScope, err = dependencies.authScope(ctx) + if err != nil { + return nil, fmt.Errorf("确定小云雀登录账号的断点作用域失败：%w", err) + } + } + if strings.TrimSpace(authScope) == "" { + return nil, fmt.Errorf("小云雀登录账号缺少可验证的断点作用域") + } bundleRoot, outputDir, exported, err := exportLibTVCanvasWithRetry( ctx, sourceURL, dependencies, prompts, stderr, ) @@ -223,10 +238,6 @@ func runCanvasImport( return nil, err } target := dependencies.target() - authScope := "" - if dependencies.authScope != nil { - authScope = dependencies.authScope() - } journalPath, err := resolveImportJournalPath( opts.JournalPath, plan.Source, @@ -273,8 +284,8 @@ func runCanvasImport( return nil, err } fmt.Fprintln(stderr, "小云雀授权在素材处理期间失效；已保留安全断点，不会重复上传，重新授权后将继续。") - if authErr := ensureCanvasImportPippitAuth( - ctx, dependencies.pippitAuth, true, prompts.promptPippitAuth, + if authErr := reauthenticateCanvasImportPippit( + ctx, dependencies.pippitAuth, prompts.promptPippitAuth, authScope, stderr, ); authErr != nil { return nil, authErr } @@ -288,13 +299,13 @@ func runCanvasImport( if prompts != nil { pippitPrompt = prompts.promptPippitAuth } - if err := ensureCanvasImportPippitAuth( - ctx, dependencies.pippitAuth, prompts != nil, pippitPrompt, + if err := ensureCanvasImportPippitAuthForScope( + ctx, dependencies.pippitAuth, prompts != nil, pippitPrompt, authScope, stderr, ); err != nil { return nil, err } result, handled, reconcileErr := reconcileExistingCanvasImport( - ctx, journalPath, plan, resolved, opts, dependencies, stderr, prompts, + ctx, journalPath, plan, resolved, opts, dependencies, stderr, prompts, authScope, ) if handled { _ = removeOwnedBundle(outputDir, bundleRoot) @@ -308,8 +319,8 @@ func runCanvasImport( if err != nil { if prompts != nil && isCanvasImportPippitAuthFailure(err) && canvasImportStateCanRetryAfterAuth(result) { fmt.Fprintln(stderr, "小云雀授权在画布处理期间失效；断点已保存，重新授权后将从安全状态继续。") - if authErr := ensureCanvasImportPippitAuth( - ctx, dependencies.pippitAuth, true, prompts.promptPippitAuth, + if authErr := reauthenticateCanvasImportPippit( + ctx, dependencies.pippitAuth, prompts.promptPippitAuth, authScope, stderr, ); authErr != nil { return result, authErr } @@ -327,8 +338,8 @@ func runCanvasImport( if result != nil && result.State == canvasplan.StateCreatePending { if prompts != nil && isCanvasImportPippitAuthFailure(errors.New(result.Warning)) { fmt.Fprintln(stderr, "小云雀授权在等待漫剧画布创建期间失效；创建请求已受理，不会重复创建，重新授权后将继续等待。") - if authErr := ensureCanvasImportPippitAuth( - ctx, dependencies.pippitAuth, true, prompts.promptPippitAuth, + if authErr := reauthenticateCanvasImportPippit( + ctx, dependencies.pippitAuth, prompts.promptPippitAuth, authScope, stderr, ); authErr != nil { return result, authErr } @@ -358,7 +369,8 @@ func canvasImportStateCanRetryAfterAuth(result *canvasplan.ExecutionResult) bool return false } switch result.State { - case canvasplan.StateCreatePending, + case canvasplan.StateInitialized, + canvasplan.StateCreatePending, canvasplan.StateRootReady, canvasplan.StateAllocationRequested, canvasplan.StateAllocated, @@ -439,6 +451,7 @@ func reconcileExistingCanvasImport( dependencies importDependencies, stderr io.Writer, prompts *importPromptSession, + expectedCredentialScope string, ) (*canvasplan.ExecutionResult, bool, error) { info, err := os.Lstat(journalPath) if os.IsNotExist(err) { @@ -464,8 +477,8 @@ func reconcileExistingCanvasImport( } if prompts != nil && isCanvasImportPippitAuthFailure(reconcileErr) { fmt.Fprintln(stderr, "小云雀授权在恢复画布断点期间失效；重新授权后将继续只读回查，不会重复提交写入。") - if authErr := ensureCanvasImportPippitAuth( - ctx, dependencies.pippitAuth, true, prompts.promptPippitAuth, + if authErr := reauthenticateCanvasImportPippit( + ctx, dependencies.pippitAuth, prompts.promptPippitAuth, expectedCredentialScope, stderr, ); authErr != nil { return result, true, authErr } @@ -590,11 +603,7 @@ func resolveImportJournalPath( return filepath.Join(directory, hex.EncodeToString(hash[:])+".journal.json"), nil } -func canvasImportAuthScope(runner *common.Runner) string { - accessKey := "" - if runner != nil && runner.Config != nil { - accessKey = strings.TrimSpace(runner.Config.AccessKey) - } +func legacyCanvasImportAuthScope(accessKey string) string { hash := sha256.Sum256([]byte(accessKey)) return hex.EncodeToString(hash[:]) } diff --git a/cmd/canvas/import_auth.go b/cmd/canvas/import_auth.go index efb1f57..027f78e 100644 --- a/cmd/canvas/import_auth.go +++ b/cmd/canvas/import_auth.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "io" "strings" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" "github.com/Pippit-dev/pippit-cli/internal/common" ) @@ -16,28 +18,59 @@ var errCanvasImportAuthCanceled = errors.New("已取消小云雀授权") var errCanvasImportReauthenticationRequired = errors.New("需要重新校验小云雀授权") type importAuthAPI interface { - AccessKey() string - SetAccessKey(string) error + AccessKey(context.Context) (string, error) + Login(context.Context, io.Writer, importAuthLoginOptions) error + HasExplicitAccessKey() bool + CredentialScope(context.Context) (string, error) Probe(context.Context) error } +type importAuthLoginOptions struct { + ForceRefresh bool + ExpectedCredentialScope string +} + type runnerImportAuthAPI struct { runner *common.Runner } -func (api runnerImportAuthAPI) AccessKey() string { - if api.runner == nil || api.runner.Config == nil { - return "" +func (api runnerImportAuthAPI) AccessKey(ctx context.Context) (string, error) { + if api.runner == nil { + return "", fmt.Errorf("小云雀 CLI 运行时配置不完整") + } + if api.runner.Auth != nil { + return api.runner.Auth.ResolveAccessKey(ctx) } - return strings.TrimSpace(api.runner.Config.AccessKey) + if api.runner.Config == nil { + return "", fmt.Errorf("小云雀 CLI 运行时配置不完整") + } + return strings.TrimSpace(api.runner.Config.AccessKey), nil } -func (api runnerImportAuthAPI) SetAccessKey(accessKey string) error { - if api.runner == nil || api.runner.Config == nil { - return fmt.Errorf("小云雀 CLI 运行时配置不完整") +func (api runnerImportAuthAPI) Login(ctx context.Context, progress io.Writer, options importAuthLoginOptions) error { + if api.runner == nil || api.runner.Auth == nil { + return fmt.Errorf("小云雀 CLI 浏览器授权尚未配置") } - api.runner.Config.AccessKey = strings.TrimSpace(accessKey) - return nil + _, err := api.runner.Auth.Login(ctx, internal_auth.LoginOptions{ + Progress: progress, + ForceRefresh: options.ForceRefresh, + ExpectedCredentialScope: options.ExpectedCredentialScope, + }) + return err +} + +func (api runnerImportAuthAPI) HasExplicitAccessKey() bool { + return api.runner != nil && api.runner.Config != nil && strings.TrimSpace(api.runner.Config.AccessKey) != "" +} + +func (api runnerImportAuthAPI) CredentialScope(ctx context.Context) (string, error) { + if api.HasExplicitAccessKey() { + return legacyCanvasImportAuthScope(strings.TrimSpace(api.runner.Config.AccessKey)), nil + } + if api.runner == nil || api.runner.Auth == nil { + return "", fmt.Errorf("小云雀 CLI 浏览器授权尚未配置") + } + return api.runner.Auth.CredentialScope(ctx) } func (api runnerImportAuthAPI) Probe(ctx context.Context) error { @@ -64,30 +97,42 @@ type importAuthPromptAction uint8 const ( importAuthPromptRetry importAuthPromptAction = iota + 1 - importAuthPromptReplace + importAuthPromptLogin importAuthPromptCancel ) type importAuthPromptRequest struct { - HasAccessKey bool - Failure string + HasCredential bool + ExplicitAccessKey bool + Failure string } type importAuthPromptResponse struct { - Action importAuthPromptAction - AccessKey string + Action importAuthPromptAction } type importAuthPrompt func(context.Context, importAuthPromptRequest) (importAuthPromptResponse, error) // ensureCanvasImportPippitAuth verifies Pippit authorization before the source -// export starts. Interactive callers may retry a transient failure, replace an -// invalid key in memory, or cancel. The key is never persisted by this flow. +// export starts. Interactive callers may retry a transient failure or complete +// browser authorization; Access Keys are never read from terminal input. func ensureCanvasImportPippitAuth( ctx context.Context, auth importAuthAPI, interactive bool, prompt importAuthPrompt, + progress ...io.Writer, +) error { + return ensureCanvasImportPippitAuthForScope(ctx, auth, interactive, prompt, "", progress...) +} + +func ensureCanvasImportPippitAuthForScope( + ctx context.Context, + auth importAuthAPI, + interactive bool, + prompt importAuthPrompt, + expectedCredentialScope string, + progress ...io.Writer, ) error { if err := ctx.Err(); err != nil { return err @@ -96,48 +141,37 @@ func ensureCanvasImportPippitAuth( return fmt.Errorf("小云雀授权检查未配置") } - accessKey := strings.TrimSpace(auth.AccessKey()) + progressWriter := io.Discard + if len(progress) > 0 && progress[0] != nil { + progressWriter = progress[0] + } failure := "" for { + accessKey, resolveErr := auth.AccessKey(ctx) + accessKey = strings.TrimSpace(accessKey) if accessKey == "" { if !interactive { - return fmt.Errorf("未找到小云雀 Access Key；请先设置 XYQ_ACCESS_KEY，或在交互模式中安全粘贴 Access Key") - } - if prompt == nil { - return fmt.Errorf("未找到小云雀 Access Key，且交互授权引导未配置") + return fmt.Errorf("未登录小云雀 CLI；请先运行 pippit-tool-cli login（CI 仍可设置 XYQ_ACCESS_KEY）") } - response, err := prompt(ctx, importAuthPromptRequest{ - HasAccessKey: false, - Failure: failure, - }) - if err != nil { - return canvasImportAuthPromptError(err) + if resolveErr != nil && !errors.Is(resolveErr, internal_auth.ErrCredentialNotFound) && + !errors.Is(resolveErr, internal_auth.ErrCredentialExpired) { + return fmt.Errorf("读取小云雀 CLI 登录凭证失败") } - switch response.Action { - case importAuthPromptCancel: - return errCanvasImportAuthCanceled - case importAuthPromptReplace: - accessKey = strings.TrimSpace(response.AccessKey) - if accessKey == "" { - failure = "Access Key 不能为空，请重新粘贴" - continue - } - if err := auth.SetAccessKey(accessKey); err != nil { - return fmt.Errorf("更新小云雀内存授权信息失败：%s", redactCanvasImportAuthFailure(err, accessKey)) - } - case importAuthPromptRetry: - failure = "当前没有可重试的 Access Key，请先粘贴" - continue - default: - return fmt.Errorf("小云雀授权引导返回了未知操作") + if err := auth.Login(ctx, progressWriter, importAuthLoginOptions{ + ForceRefresh: expectedCredentialScope != "", + ExpectedCredentialScope: expectedCredentialScope, + }); err != nil { + return fmt.Errorf("小云雀网页授权失败：%w", err) } + failure = "" + continue } if err := ctx.Err(); err != nil { return err } if err := auth.Probe(ctx); err == nil { - return nil + return verifyCanvasImportCredentialScope(ctx, auth, expectedCredentialScope) } else { if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr @@ -152,8 +186,9 @@ func ensureCanvasImportPippitAuth( } response, err := prompt(ctx, importAuthPromptRequest{ - HasAccessKey: true, - Failure: failure, + HasCredential: true, + ExplicitAccessKey: auth.HasExplicitAccessKey(), + Failure: failure, }) if err != nil { return canvasImportAuthPromptError(err) @@ -161,17 +196,17 @@ func ensureCanvasImportPippitAuth( switch response.Action { case importAuthPromptRetry: continue - case importAuthPromptReplace: - replacement := strings.TrimSpace(response.AccessKey) - if replacement == "" { - accessKey = "" - failure = "Access Key 不能为空，请重新粘贴" - continue + case importAuthPromptLogin: + if auth.HasExplicitAccessKey() { + return fmt.Errorf("当前 XYQ_ACCESS_KEY 会覆盖浏览器登录；请先取消导入并在 shell 中 unset XYQ_ACCESS_KEY") } - if err := auth.SetAccessKey(replacement); err != nil { - return fmt.Errorf("更新小云雀内存授权信息失败：%s", redactCanvasImportAuthFailure(err, replacement)) + if err := auth.Login(ctx, progressWriter, importAuthLoginOptions{ + ForceRefresh: true, + ExpectedCredentialScope: expectedCredentialScope, + }); err != nil { + return fmt.Errorf("小云雀网页授权失败：%w", err) } - accessKey = replacement + failure = "" case importAuthPromptCancel: return errCanvasImportAuthCanceled default: @@ -180,6 +215,45 @@ func ensureCanvasImportPippitAuth( } } +func reauthenticateCanvasImportPippit( + ctx context.Context, + auth importAuthAPI, + prompt importAuthPrompt, + expectedCredentialScope string, + progress io.Writer, +) error { + if auth == nil { + return fmt.Errorf("小云雀授权检查未配置") + } + if auth.HasExplicitAccessKey() { + return fmt.Errorf("当前 XYQ_ACCESS_KEY 会覆盖浏览器登录；请取消导入并在 shell 中 unset XYQ_ACCESS_KEY") + } + if err := auth.Login(ctx, progress, importAuthLoginOptions{ + ForceRefresh: true, + ExpectedCredentialScope: expectedCredentialScope, + }); err != nil { + return fmt.Errorf("小云雀网页重新授权失败：%w", err) + } + return ensureCanvasImportPippitAuthForScope( + ctx, auth, true, prompt, expectedCredentialScope, progress, + ) +} + +func verifyCanvasImportCredentialScope(ctx context.Context, auth importAuthAPI, expected string) error { + expected = strings.TrimSpace(expected) + if expected == "" { + return nil + } + actual, err := auth.CredentialScope(ctx) + if err != nil { + return fmt.Errorf("重新授权后无法确认小云雀账号：%w", err) + } + if strings.TrimSpace(actual) != expected { + return fmt.Errorf("%w；为避免复用上一账号的素材或画布断点，本次导入已安全停止", internal_auth.ErrCredentialAccountMismatch) + } + return nil +} + func canvasImportAuthPromptError(err error) error { if err == nil { return nil @@ -211,7 +285,8 @@ func redactCanvasImportFinalError(err error, auth importAuthAPI) error { return err } original := err.Error() - redacted := redactCanvasImportAuthFailure(err, auth.AccessKey()) + accessKey, _ := auth.AccessKey(context.Background()) + redacted := redactCanvasImportAuthFailure(err, accessKey) if redacted == original { return err } @@ -236,7 +311,9 @@ func isCanvasImportPippitAuthFailure(err error) bool { if err == nil { return false } - if errors.Is(err, errCanvasImportReauthenticationRequired) { + if errors.Is(err, errCanvasImportReauthenticationRequired) || + errors.Is(err, internal_auth.ErrCredentialNotFound) || + errors.Is(err, internal_auth.ErrCredentialExpired) { return true } message := strings.ToLower(err.Error()) diff --git a/cmd/canvas/import_auth_flow.go b/cmd/canvas/import_auth_flow.go index 31146bf..43b29c1 100644 --- a/cmd/canvas/import_auth_flow.go +++ b/cmd/canvas/import_auth_flow.go @@ -7,8 +7,6 @@ import ( "strings" ) -const pippitAccessKeySettingsURL = "https://xyq.jianying.com/home?tab_name=home" - func preflightCanvasImportAuth( ctx context.Context, dependencies importDependencies, @@ -22,7 +20,7 @@ func preflightCanvasImportAuth( } fmt.Fprintln(stderr, "阶段：正在检查小云雀授权…") - if err := ensureCanvasImportPippitAuth(ctx, dependencies.pippitAuth, interactive, pippitPrompt); err != nil { + if err := ensureCanvasImportPippitAuth(ctx, dependencies.pippitAuth, interactive, pippitPrompt, stderr); err != nil { return err } fmt.Fprintln(stderr, "小云雀授权校验通过。") @@ -76,51 +74,40 @@ func (prompts *importPromptSession) promptPippitAuth( if strings.TrimSpace(request.Failure) != "" { fmt.Fprintf(prompts.stderr, "小云雀授权提示：%s\n", request.Failure) } - if !request.HasAccessKey { - fmt.Fprintf( - prompts.stderr, - "未检测到小云雀 Access Key。请先在个人设置页创建或查看：%s\n"+ - "随后在下方粘贴；它只保存在当前 CLI 进程内，不会写入配置、日志或断点记录。\n", - pippitAccessKeySettingsURL, + if request.ExplicitAccessKey { + choice, err := prompts.askChoice( + "当前 XYQ_ACCESS_KEY 校验失败：", + []importPromptChoice{ + {label: "重新校验当前环境变量"}, + {label: "取消导入并在 shell 中取消 XYQ_ACCESS_KEY（默认）"}, + }, + 2, ) - accessKey, eof, err := prompts.readSecret("粘贴小云雀 Access Key：") if err != nil { return importAuthPromptResponse{}, err } - if eof && strings.TrimSpace(accessKey) == "" { - return importAuthPromptResponse{Action: importAuthPromptCancel}, nil + if choice == 1 { + return importAuthPromptResponse{Action: importAuthPromptRetry}, nil } - return importAuthPromptResponse{Action: importAuthPromptReplace, AccessKey: accessKey}, nil - } - - defaultChoice := 1 - if strings.Contains(request.Failure, "401") || strings.Contains(request.Failure, "403") { - defaultChoice = 2 + return importAuthPromptResponse{Action: importAuthPromptCancel}, nil } choice, err := prompts.askChoice( "小云雀授权下一步：", []importPromptChoice{ - {label: "重新校验当前 Access Key"}, - {label: "粘贴新的 Access Key"}, + {label: "重新打开浏览器授权（默认）"}, + {label: "重新校验当前登录"}, {label: "取消导入"}, }, - defaultChoice, + 1, ) if err != nil { return importAuthPromptResponse{}, err } switch choice { case 1: - return importAuthPromptResponse{Action: importAuthPromptRetry}, nil + return importAuthPromptResponse{Action: importAuthPromptLogin}, nil case 2: - accessKey, eof, readErr := prompts.readSecret("粘贴新的小云雀 Access Key：") - if readErr != nil { - return importAuthPromptResponse{}, readErr - } - if eof && strings.TrimSpace(accessKey) == "" { - return importAuthPromptResponse{Action: importAuthPromptCancel}, nil - } - return importAuthPromptResponse{Action: importAuthPromptReplace, AccessKey: accessKey}, nil + return importAuthPromptResponse{Action: importAuthPromptRetry}, nil default: return importAuthPromptResponse{Action: importAuthPromptCancel}, nil } diff --git a/cmd/canvas/import_auth_flow_test.go b/cmd/canvas/import_auth_flow_test.go index 57766fc..3712899 100644 --- a/cmd/canvas/import_auth_flow_test.go +++ b/cmd/canvas/import_auth_flow_test.go @@ -4,31 +4,63 @@ import ( "bytes" "context" "errors" + "fmt" "io" "path/filepath" "strings" "testing" "time" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" "github.com/Pippit-dev/pippit-cli/internal/canvasplan" ) type trackingImportAuthAPI struct { - key string - events *[]string - probeErrors []error - setValues []string + key string + loginKey string + events *[]string + probeErrors []error + loginErrors []error + logins int + explicit bool + credentialScope string + loginScopes []string + loginOptions []importAuthLoginOptions } -func (auth *trackingImportAuthAPI) AccessKey() string { return auth.key } +func (auth *trackingImportAuthAPI) AccessKey(context.Context) (string, error) { return auth.key, nil } -func (auth *trackingImportAuthAPI) SetAccessKey(value string) error { - auth.key = strings.TrimSpace(value) - auth.setValues = append(auth.setValues, auth.key) +func (auth *trackingImportAuthAPI) Login(_ context.Context, _ io.Writer, options importAuthLoginOptions) error { + auth.logins++ + auth.loginOptions = append(auth.loginOptions, options) + if len(auth.loginErrors) > 0 { + err := auth.loginErrors[0] + auth.loginErrors = auth.loginErrors[1:] + if err != nil { + return err + } + } + if auth.loginKey == "" { + auth.loginKey = "browser-managed-key" + } + auth.key = auth.loginKey + if len(auth.loginScopes) > 0 { + auth.credentialScope = auth.loginScopes[0] + auth.loginScopes = auth.loginScopes[1:] + } return nil } +func (auth *trackingImportAuthAPI) HasExplicitAccessKey() bool { return auth.explicit } + +func (auth *trackingImportAuthAPI) CredentialScope(context.Context) (string, error) { + if auth.credentialScope != "" { + return auth.credentialScope, nil + } + return "browser-device-scope", nil +} + func (auth *trackingImportAuthAPI) Probe(context.Context) error { if auth.events != nil { *auth.events = append(*auth.events, "pippit-auth") @@ -67,17 +99,28 @@ type trackingImportExporter struct { } type expiringImportMediaAPI struct { - uploads int + uploads int + uploadSucceeds bool + preflights int + preflightErrors []error } -func (api *expiringImportMediaAPI) PreflightUpload(context.Context) error { return nil } +func (api *expiringImportMediaAPI) PreflightUpload(context.Context) error { + api.preflights++ + if len(api.preflightErrors) == 0 { + return nil + } + err := api.preflightErrors[0] + api.preflightErrors = api.preflightErrors[1:] + return err +} func (api *expiringImportMediaAPI) Upload( context.Context, validatedImportMedia, ) (*canvascore.UploadResult, error) { api.uploads++ - if api.uploads == 1 { + if api.uploads == 1 && !api.uploadSucceeds { return nil, errors.New("HTTP 401") } return &canvascore.UploadResult{ @@ -216,7 +259,7 @@ func TestCanvasImportMissingPippitKeyStopsBeforeLibTVOrFilesystemSideEffects(t * _, err := runCanvasImport(context.Background(), importOptions{ Provider: "libtv", SourceURL: testLibTVURL, }, deps, io.Discard, nil) - if err == nil || !strings.Contains(err.Error(), "未找到小云雀 Access Key") { + if err == nil || !strings.Contains(err.Error(), "pippit-tool-cli login") { t.Fatalf("runCanvasImport() error = %v, want missing-key guidance", err) } if sourceAuth.calls != 0 || len(exporter.inner.urls) != 0 || cacheTouched { @@ -224,14 +267,13 @@ func TestCanvasImportMissingPippitKeyStopsBeforeLibTVOrFilesystemSideEffects(t * } } -func TestCanvasImportAuthPromptsForPippitKeyThenChecksLibTV(t *testing.T) { - const accessKey = "pasted-secret-access-key" +func TestCanvasImportAuthOpensPippitBrowserLoginThenChecksLibTV(t *testing.T) { events := []string{} - pippit := &trackingImportAuthAPI{events: &events} + pippit := &trackingImportAuthAPI{events: &events, loginKey: "browser-managed-key"} source := &trackingSourceAuthenticator{events: &events} var stderr bytes.Buffer prompts := newImportPromptSessionWithTUI( - context.Background(), strings.NewReader(accessKey+"\n"), &stderr, false, + context.Background(), strings.NewReader(""), &stderr, false, ) err := preflightCanvasImportAuth(context.Background(), importDependencies{ @@ -244,24 +286,25 @@ func TestCanvasImportAuthPromptsForPippitKeyThenChecksLibTV(t *testing.T) { if got := strings.Join(events, ","); got != "pippit-auth,libtv-auth" { t.Fatalf("auth order = %q, want Pippit then LibTV", got) } - if pippit.key != accessKey || len(pippit.setValues) != 1 { - t.Fatalf("in-memory Access Key = %q / %v", pippit.key, pippit.setValues) + if pippit.key != "browser-managed-key" || pippit.logins != 1 { + t.Fatalf("browser credential/logins = %q/%d", pippit.key, pippit.logins) } - if strings.Contains(stderr.String(), accessKey) { - t.Fatalf("stderr leaked pasted Access Key: %q", stderr.String()) + if strings.Contains(stderr.String(), pippit.key) { + t.Fatalf("stderr leaked browser-managed Access Key: %q", stderr.String()) } } -func TestCanvasImportAuthReplacesRejectedPippitKeyWithoutLeakingIt(t *testing.T) { +func TestCanvasImportAuthReauthorizesRejectedManagedCredentialWithoutLeakingIt(t *testing.T) { const oldKey = "rejected-secret-key" - const newKey = "replacement-secret-key" + const newKey = "browser-replacement-key" pippit := &trackingImportAuthAPI{ key: oldKey, + loginKey: newKey, probeErrors: []error{errors.New("HTTP 401 " + oldKey), nil}, } var stderr bytes.Buffer prompts := newImportPromptSessionWithTUI( - context.Background(), strings.NewReader("2\n"+newKey+"\n"), &stderr, false, + context.Background(), strings.NewReader("1\n"), &stderr, false, ) err := preflightCanvasImportAuth(context.Background(), importDependencies{ @@ -271,8 +314,8 @@ func TestCanvasImportAuthReplacesRejectedPippitKeyWithoutLeakingIt(t *testing.T) if err != nil { t.Fatalf("preflightCanvasImportAuth() error = %v", err) } - if pippit.key != newKey { - t.Fatalf("Access Key = %q, want replacement", pippit.key) + if pippit.key != newKey || pippit.logins != 1 { + t.Fatalf("browser credential/logins = %q/%d, want replacement once", pippit.key, pippit.logins) } if strings.Contains(stderr.String(), oldKey) || strings.Contains(stderr.String(), newKey) { t.Fatalf("stderr leaked an Access Key: %q", stderr.String()) @@ -377,6 +420,66 @@ func TestCanvasImportReauthorizesDuringMediaWithoutBlindUploadReplay(t *testing. } } +func TestCanvasImportReauthorizesWhenCredentialExpiresAfterExport(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &expiringImportMediaAPI{uploadSucceeds: true, preflightErrors: []error{ + fmt.Errorf("credential vanished after export: %w", internal_auth.ErrCredentialExpired), + nil, + }} + pippit := &trackingImportAuthAPI{key: "expired-key", loginKey: "replacement-key"} + deps := testImportDependencies(temp, exporter, media, &fakeImportExecutor{result: verifiedImportResult()}) + deps.pippitAuth = pippit + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI(context.Background(), strings.NewReader(""), &stderr, false) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, &stderr, prompts) + if err != nil { + t.Fatalf("runCanvasImport() error = %v", err) + } + if result == nil || result.State != canvasplan.StateVerified || media.uploads != 1 || pippit.logins != 1 { + t.Fatalf("result/uploads/logins = %#v/%d/%d", result, media.uploads, pippit.logins) + } + if len(pippit.loginOptions) != 1 || !pippit.loginOptions[0].ForceRefresh || + pippit.loginOptions[0].ExpectedCredentialScope != "browser-device-scope" { + t.Fatalf("login options = %#v, want task-bound forced refresh", pippit.loginOptions) + } +} + +func TestCanvasImportRejectsDifferentAccountDuringReauthentication(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + media := &expiringImportMediaAPI{} + pippit := &trackingImportAuthAPI{ + key: "account-a-key", + loginKey: "account-b-key", + credentialScope: "account-a-scope", + loginScopes: []string{"account-b-scope"}, + } + deps := testImportDependencies(temp, exporter, media, &fakeImportExecutor{result: verifiedImportResult()}) + deps.pippitAuth = pippit + deps.authScope = pippit.CredentialScope + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI(context.Background(), strings.NewReader(""), &stderr, false) + + _, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, + }, deps, &stderr, prompts) + if !errors.Is(err, internal_auth.ErrCredentialAccountMismatch) { + t.Fatalf("runCanvasImport() error = %v, want account mismatch", err) + } + if media.uploads != 1 || pippit.logins != 1 { + t.Fatalf("uploads/logins = %d/%d, want stop immediately after first rejected request and reauth", media.uploads, pippit.logins) + } + if len(pippit.loginOptions) != 1 || pippit.loginOptions[0].ExpectedCredentialScope != "account-a-scope" { + t.Fatalf("login options = %#v, want account-a binding", pippit.loginOptions) + } +} + func TestCanvasImportWaitsForAcceptedCreateInSameProcess(t *testing.T) { temp := t.TempDir() plan, mediaBytes := testImportPlan(t, false) @@ -416,7 +519,8 @@ func TestCanvasImportReauthorizesWhileWaitingForAcceptedCreate(t *testing.T) { verifiedImportResult(), }} pippit := &trackingImportAuthAPI{ - key: "expired-key", + key: "expired-key", + loginKey: "replacement-key", probeErrors: []error{ nil, nil, @@ -429,7 +533,7 @@ func TestCanvasImportReauthorizesWhileWaitingForAcceptedCreate(t *testing.T) { deps.mediaPoll = time.Millisecond var stderr bytes.Buffer prompts := newImportPromptSessionWithTUI( - context.Background(), strings.NewReader("2\nreplacement-key\n"), &stderr, false, + context.Background(), strings.NewReader("1\n"), &stderr, false, ) result, err := runCanvasImport(context.Background(), importOptions{ @@ -458,7 +562,8 @@ func TestCanvasImportReauthorizesAmbiguousApplyThenQueriesWithoutReplay(t *testi errors: []error{errors.New("canvas apply failed; exact query-back failed: HTTP 401")}, } pippit := &trackingImportAuthAPI{ - key: "expired-key", + key: "expired-key", + loginKey: "replacement-key", probeErrors: []error{ nil, nil, @@ -470,7 +575,7 @@ func TestCanvasImportReauthorizesAmbiguousApplyThenQueriesWithoutReplay(t *testi deps.pippitAuth = pippit var stderr bytes.Buffer prompts := newImportPromptSessionWithTUI( - context.Background(), strings.NewReader("2\nreplacement-key\n"), &stderr, false, + context.Background(), strings.NewReader("1\n"), &stderr, false, ) result, err := runCanvasImport(context.Background(), importOptions{ @@ -530,7 +635,8 @@ func TestCanvasImportReauthorizesExistingAmbiguousJournal(t *testing.T) { errors: []error{errors.New("query assets failed: ret=1015")}, } pippit := &trackingImportAuthAPI{ - key: "expired-key", + key: "expired-key", + loginKey: "replacement-key", probeErrors: []error{ nil, nil, @@ -542,7 +648,7 @@ func TestCanvasImportReauthorizesExistingAmbiguousJournal(t *testing.T) { deps.pippitAuth = pippit var stderr bytes.Buffer prompts := newImportPromptSessionWithTUI( - context.Background(), strings.NewReader("2\nreplacement-key\n"), &stderr, false, + context.Background(), strings.NewReader("1\n"), &stderr, false, ) result, err := runCanvasImport(context.Background(), importOptions{ diff --git a/cmd/canvas/import_auth_test.go b/cmd/canvas/import_auth_test.go index 7d35251..9793621 100644 --- a/cmd/canvas/import_auth_test.go +++ b/cmd/canvas/import_auth_test.go @@ -5,37 +5,80 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "net/http/httptest" "strings" "testing" "time" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/common" "github.com/Pippit-dev/pippit-cli/internal/config" ) type fakeImportAuthAPI struct { - accessKey string - setValues []string - probeErrors []error - probes int - setErr error + accessKey string + loginKey string + probeErrors []error + probes int + logins int + loginErr error + resolveErr error + explicit bool + credentialScope string + loginOptions []importAuthLoginOptions } -func (api *fakeImportAuthAPI) AccessKey() string { - return api.accessKey +type credentialErrorAuthManager struct{ err error } + +func (manager credentialErrorAuthManager) ResolveAccessKey(context.Context) (string, error) { + return "", manager.err +} + +func (credentialErrorAuthManager) Login(context.Context, internal_auth.LoginOptions) (*internal_auth.Credential, error) { + return nil, errors.New("unexpected login") +} + +func (credentialErrorAuthManager) Status(context.Context) (*internal_auth.Status, error) { + return nil, errors.New("unexpected status") +} + +func (credentialErrorAuthManager) Logout(context.Context, bool) error { + return errors.New("unexpected logout") +} + +func (credentialErrorAuthManager) CredentialScope(context.Context) (string, error) { + return "", errors.New("unexpected scope") +} + +func (api *fakeImportAuthAPI) AccessKey(context.Context) (string, error) { + return api.accessKey, api.resolveErr } -func (api *fakeImportAuthAPI) SetAccessKey(accessKey string) error { - api.setValues = append(api.setValues, accessKey) - if api.setErr != nil { - return api.setErr +func (api *fakeImportAuthAPI) Login(_ context.Context, _ io.Writer, options importAuthLoginOptions) error { + api.logins++ + api.loginOptions = append(api.loginOptions, options) + if api.loginErr != nil { + return api.loginErr } - api.accessKey = accessKey + if strings.TrimSpace(api.loginKey) == "" { + api.loginKey = "browser-managed-key" + } + api.accessKey = api.loginKey + api.resolveErr = nil return nil } +func (api *fakeImportAuthAPI) HasExplicitAccessKey() bool { return api.explicit } + +func (api *fakeImportAuthAPI) CredentialScope(context.Context) (string, error) { + if api.credentialScope == "" { + return "browser-device-scope", nil + } + return api.credentialScope, nil +} + func (api *fakeImportAuthAPI) Probe(context.Context) error { api.probes++ if len(api.probeErrors) == 0 { @@ -62,8 +105,8 @@ func TestEnsureCanvasImportPippitAuthAcceptsExistingKey(t *testing.T) { if err != nil { t.Fatalf("ensureCanvasImportPippitAuth() error = %v", err) } - if auth.probes != 1 || prompted || len(auth.setValues) != 0 { - t.Fatalf("probes/prompted/sets = %d/%v/%v, want 1/false/none", auth.probes, prompted, auth.setValues) + if auth.probes != 1 || prompted || auth.logins != 0 { + t.Fatalf("probes/prompted/logins = %d/%v/%d, want 1/false/0", auth.probes, prompted, auth.logins) } } @@ -71,48 +114,36 @@ func TestEnsureCanvasImportPippitAuthMissingKeyIsSideEffectFreeWhenNonInteractiv auth := &fakeImportAuthAPI{} err := ensureCanvasImportPippitAuth(context.Background(), auth, false, nil) - if err == nil || !strings.Contains(err.Error(), "未找到小云雀 Access Key") { + if err == nil || !strings.Contains(err.Error(), "pippit-tool-cli login") { t.Fatalf("ensureCanvasImportPippitAuth() error = %v, want missing-key guidance", err) } - if auth.probes != 0 || len(auth.setValues) != 0 { - t.Fatalf("probes/sets = %d/%v, want no side effects", auth.probes, auth.setValues) + if auth.probes != 0 || auth.logins != 0 { + t.Fatalf("probes/logins = %d/%d, want no side effects", auth.probes, auth.logins) } } -func TestEnsureCanvasImportPippitAuthPromptsForMissingKeyWithoutProbingEmptyInput(t *testing.T) { - auth := &fakeImportAuthAPI{} - responses := []importAuthPromptResponse{ - {Action: importAuthPromptReplace, AccessKey: " "}, - {Action: importAuthPromptReplace, AccessKey: " pasted-key "}, - } - requests := make([]importAuthPromptRequest, 0, len(responses)) +func TestEnsureCanvasImportPippitAuthMissingCredentialStartsBrowserLoginBeforeProbe(t *testing.T) { + auth := &fakeImportAuthAPI{loginKey: "browser-key"} + prompted := false err := ensureCanvasImportPippitAuth( context.Background(), auth, true, - func(_ context.Context, request importAuthPromptRequest) (importAuthPromptResponse, error) { - requests = append(requests, request) - response := responses[0] - responses = responses[1:] - return response, nil + func(context.Context, importAuthPromptRequest) (importAuthPromptResponse, error) { + prompted = true + return importAuthPromptResponse{}, nil }, ) if err != nil { t.Fatalf("ensureCanvasImportPippitAuth() error = %v", err) } - if len(requests) != 2 || requests[0].HasAccessKey || requests[1].HasAccessKey { - t.Fatalf("prompt requests = %#v, want two missing-key prompts", requests) - } - if !strings.Contains(requests[1].Failure, "不能为空") { - t.Fatalf("second prompt failure = %q, want empty-key guidance", requests[1].Failure) - } - if auth.probes != 1 || strings.Join(auth.setValues, ",") != "pasted-key" { - t.Fatalf("probes/sets = %d/%v, want one verified in-memory update", auth.probes, auth.setValues) + if auth.logins != 1 || auth.probes != 1 || prompted { + t.Fatalf("logins/probes/prompted = %d/%d/%v, want 1/1/false", auth.logins, auth.probes, prompted) } } -func TestEnsureCanvasImportPippitAuthRetriesAndReplacesInvalidKey(t *testing.T) { +func TestEnsureCanvasImportPippitAuthRetriesThenUsesBrowserLogin(t *testing.T) { auth := &fakeImportAuthAPI{ accessKey: "invalid-secret-key", probeErrors: []error{ @@ -123,7 +154,7 @@ func TestEnsureCanvasImportPippitAuthRetriesAndReplacesInvalidKey(t *testing.T) } responses := []importAuthPromptResponse{ {Action: importAuthPromptRetry}, - {Action: importAuthPromptReplace, AccessKey: "replacement-key"}, + {Action: importAuthPromptLogin}, } requests := make([]importAuthPromptRequest, 0, len(responses)) @@ -141,10 +172,13 @@ func TestEnsureCanvasImportPippitAuthRetriesAndReplacesInvalidKey(t *testing.T) if err != nil { t.Fatalf("ensureCanvasImportPippitAuth() error = %v", err) } - if auth.probes != 3 || strings.Join(auth.setValues, ",") != "replacement-key" { - t.Fatalf("probes/sets = %d/%v, want retry then replacement", auth.probes, auth.setValues) + if auth.probes != 3 || auth.logins != 1 { + t.Fatalf("probes/logins = %d/%d, want retry then browser login", auth.probes, auth.logins) } - if len(requests) != 2 || !requests[0].HasAccessKey || !requests[1].HasAccessKey { + if len(auth.loginOptions) != 1 || !auth.loginOptions[0].ForceRefresh { + t.Fatalf("login options = %#v, want forced replacement after rejected AK", auth.loginOptions) + } + if len(requests) != 2 || !requests[0].HasCredential || !requests[1].HasCredential { t.Fatalf("prompt requests = %#v, want failed-key prompts", requests) } if strings.Contains(requests[0].Failure, "invalid-secret-key") || !strings.Contains(requests[0].Failure, "HTTP 401") { @@ -169,8 +203,8 @@ func TestEnsureCanvasImportPippitAuthCanCancelAfterProbeFailure(t *testing.T) { if !errors.Is(err, errCanvasImportAuthCanceled) { t.Fatalf("ensureCanvasImportPippitAuth() error = %v, want cancellation", err) } - if auth.probes != 1 || len(auth.setValues) != 0 { - t.Fatalf("probes/sets = %d/%v, want one read-only probe and no update", auth.probes, auth.setValues) + if auth.probes != 1 || auth.logins != 0 { + t.Fatalf("probes/logins = %d/%d, want one read-only probe and no login", auth.probes, auth.logins) } } @@ -179,7 +213,7 @@ func TestEnsureCanvasImportPippitAuthDoesNotExposePromptErrorText(t *testing.T) err := ensureCanvasImportPippitAuth( context.Background(), - &fakeImportAuthAPI{}, + &fakeImportAuthAPI{accessKey: "invalid-key", probeErrors: []error{errors.New("HTTP 401")}}, true, func(context.Context, importAuthPromptRequest) (importAuthPromptResponse, error) { return importAuthPromptResponse{}, errors.New("failed after reading " + candidate) @@ -231,7 +265,7 @@ func TestRunnerImportAuthAPIUsesReadOnlyCanvasQueryAndMemoryOnlyKey(t *testing.T })) defer server.Close() - cfg := &config.Config{BaseURL: server.URL, HTTPTimeout: time.Second} + cfg := &config.Config{BaseURL: server.URL, HTTPTimeout: time.Second, AccessKey: "pasted-key"} runner := common.NewRunner(cfg, nil) runner.Client = common.NewHTTPClient( cfg.BaseURL, @@ -240,17 +274,21 @@ func TestRunnerImportAuthAPIUsesReadOnlyCanvasQueryAndMemoryOnlyKey(t *testing.T ) auth := runnerImportAuthAPI{runner: runner} - if err := auth.SetAccessKey(" pasted-key "); err != nil { - t.Fatalf("SetAccessKey() error = %v", err) - } if err := auth.Probe(context.Background()); err != nil { t.Fatalf("Probe() error = %v", err) } if method != http.MethodPost || strings.Join(assetIDs, ",") != "9223372036854775807" { t.Fatalf("probe method/assets = %q/%v, want read-only Canvas query sentinel 9223372036854775807", method, assetIDs) } - if cfg.AccessKey != "pasted-key" { - t.Fatalf("Config.AccessKey = %q, want trimmed in-memory key", cfg.AccessKey) +} + +func TestRunnerImportMediaPreflightPreservesCredentialState(t *testing.T) { + for _, cause := range []error{internal_auth.ErrCredentialNotFound, internal_auth.ErrCredentialExpired} { + api := runnerImportMediaAPI{runner: &common.Runner{Auth: credentialErrorAuthManager{err: cause}}} + err := api.PreflightUpload(context.Background()) + if !errors.Is(err, cause) { + t.Fatalf("PreflightUpload() error = %v, want wrapped %v", err, cause) + } } } @@ -292,3 +330,12 @@ func TestCanvasImportPippitAuthFailureRecognizesBusinessRetCode(t *testing.T) { } } } + +func TestCanvasImportPippitAuthFailureRecognizesWrappedCredentialState(t *testing.T) { + for _, cause := range []error{internal_auth.ErrCredentialNotFound, internal_auth.ErrCredentialExpired} { + err := fmt.Errorf("upload preflight: %w", cause) + if !isCanvasImportPippitAuthFailure(err) { + t.Fatalf("wrapped credential error was not recognized: %v", err) + } + } +} diff --git a/cmd/canvas/import_media.go b/cmd/canvas/import_media.go index a043da1..80d3125 100644 --- a/cmd/canvas/import_media.go +++ b/cmd/canvas/import_media.go @@ -83,11 +83,21 @@ func (api runnerImportMediaAPI) PreflightUpload(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } - if api.runner == nil || api.runner.Config == nil { + if api.runner == nil { return fmt.Errorf("画布素材上传器尚未配置") } - if strings.TrimSpace(api.runner.Config.AccessKey) == "" { - return fmt.Errorf("缺少 XYQ_ACCESS_KEY；请先完成小云雀 CLI 授权，再导入素材") + accessKey := "" + if api.runner.Auth != nil { + resolved, err := api.runner.Auth.ResolveAccessKey(ctx) + if err != nil { + return fmt.Errorf("未找到可用的小云雀 CLI 登录凭证；请先运行 pippit-tool-cli login: %w", err) + } + accessKey = resolved + } else if api.runner.Config != nil { + accessKey = api.runner.Config.AccessKey + } + if strings.TrimSpace(accessKey) == "" { + return fmt.Errorf("未登录小云雀 CLI；请先运行 pippit-tool-cli login") } return nil } diff --git a/cmd/canvas/import_prompt.go b/cmd/canvas/import_prompt.go index 48e7d2a..fde5eff 100644 --- a/cmd/canvas/import_prompt.go +++ b/cmd/canvas/import_prompt.go @@ -7,8 +7,6 @@ import ( "io" "os" "strings" - - charmterm "github.com/charmbracelet/x/term" ) const importFlagsHint = `--from libtv --url "https://www.liblib.tv/canvas?projectId=<project-id>"` @@ -166,23 +164,6 @@ func prepareCanvasImportOptions( return opts, prompts, nil } -func (prompts *importPromptSession) readSecret(label string) (string, bool, error) { - if prompts.tui != nil { - value, err := prompts.tui.readSecret(label) - return value, false, err - } - if file, ok := prompts.input.(*os.File); ok && importFileIsTerminal(file) { - fmt.Fprint(prompts.stderr, label) - value, err := charmterm.ReadPassword(file.Fd()) - fmt.Fprintln(prompts.stderr) - if err != nil { - return "", false, fmt.Errorf("安全读取 Access Key 失败：%w", err) - } - return strings.TrimSpace(string(value)), false, nil - } - return prompts.readLine(label) -} - func (prompts *importPromptSession) askChoice( title string, choices []importPromptChoice, diff --git a/cmd/canvas/import_prompt_tui.go b/cmd/canvas/import_prompt_tui.go index 92f2e15..f4b9028 100644 --- a/cmd/canvas/import_prompt_tui.go +++ b/cmd/canvas/import_prompt_tui.go @@ -74,26 +74,6 @@ func (prompt *importPromptTUI) readLine(label string) (string, error) { return strings.TrimSpace(value), nil } -func (prompt *importPromptTUI) readSecret(label string) (string, error) { - value := "" - field := huh.NewInput(). - Title(strings.TrimSpace(strings.TrimRight(label, ":： "))). - Description("粘贴后按 Enter 确认；内容仅用于当前进程，不会保存"). - Prompt("› "). - EchoMode(huh.EchoModePassword). - Validate(func(value string) error { - if strings.TrimSpace(value) == "" { - return fmt.Errorf("Access Key 不能为空") - } - return nil - }). - Value(&value) - if err := prompt.run(field); err != nil { - return "", err - } - return strings.TrimSpace(value), nil -} - func (prompt *importPromptTUI) run(field huh.Field) error { form := huh.NewForm(huh.NewGroup(field)). WithTheme(huh.ThemeCharm()). diff --git a/cmd/canvas/import_prompt_tui_test.go b/cmd/canvas/import_prompt_tui_test.go index b900305..566b8fa 100644 --- a/cmd/canvas/import_prompt_tui_test.go +++ b/cmd/canvas/import_prompt_tui_test.go @@ -81,30 +81,6 @@ func TestImportPromptTUIReadsPastedURL(t *testing.T) { } } -func TestImportPromptTUIKeepsAccessKeyMasked(t *testing.T) { - t.Setenv("TERM", "xterm-256color") - const accessKey = "tui-secret-access-key" - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - var output bytes.Buffer - session := newImportPromptSessionWithTUI( - ctx, - &delayedImportPromptReader{Reader: bytes.NewBufferString(accessKey + "\r")}, - &output, - true, - ) - value, eof, err := session.readSecret("粘贴小云雀 Access Key：") - if err != nil { - t.Fatalf("readSecret() error = %v; output = %q", err, output.String()) - } - if eof || value != accessKey { - t.Fatalf("readSecret() = %q/%v, want masked value", value, eof) - } - if strings.Contains(output.String(), accessKey) { - t.Fatalf("TUI output leaked Access Key: %q", output.String()) - } -} - func TestImportPromptTUIHonorsContextCancellation(t *testing.T) { t.Setenv("TERM", "xterm-256color") ctx, cancel := context.WithCancel(context.Background()) diff --git a/cmd/canvas/import_test.go b/cmd/canvas/import_test.go index 090e43b..3e5b31f 100644 --- a/cmd/canvas/import_test.go +++ b/cmd/canvas/import_test.go @@ -22,8 +22,6 @@ import ( canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" "github.com/Pippit-dev/pippit-cli/internal/canvasplan" - "github.com/Pippit-dev/pippit-cli/internal/common" - "github.com/Pippit-dev/pippit-cli/internal/config" ) type fakeImportExporter struct { @@ -953,15 +951,15 @@ func TestMediaCheckpointDoesNotMarkMissingAKAsUploadRequested(t *testing.T) { } } -func TestDefaultJournalPathSeparatesPippitAccessKeys(t *testing.T) { +func TestLegacyCanvasImportAuthScopeSeparatesExplicitAccessKeys(t *testing.T) { configDirectory := t.TempDir() configDir := func() (string, error) { return configDirectory, nil } source := canvasplan.Source{ Provider: "libtv", ProjectID: "037a5c49e1b344e5adbc899ad93fdca9", Fingerprint: "sha256:" + strings.Repeat("1", 64), } target := "https://xyq.jianying.com|ppe_cli_canvas_ak" - firstScope := canvasImportAuthScope(&common.Runner{Config: &config.Config{AccessKey: "first-account-ak"}}) - secondScope := canvasImportAuthScope(&common.Runner{Config: &config.Config{AccessKey: "second-account-ak"}}) + firstScope := legacyCanvasImportAuthScope("first-account-ak") + secondScope := legacyCanvasImportAuthScope("second-account-ak") firstPath, err := resolveImportJournalPath("", source, target, firstScope, configDir) if err != nil { t.Fatal(err) @@ -1125,9 +1123,11 @@ func testImportDependencies( userCacheDir: func() (string, error) { return filepath.Join(root, "cache"), nil }, userConfigDir: func() (string, error) { return filepath.Join(root, "config"), nil }, target: func() string { return "https://xyq.jianying.com|ppe_cli_canvas_ak" }, - authScope: func() string { return strings.Repeat("a", 64) }, - mediaPoll: time.Millisecond, - mediaTimeout: time.Second, + authScope: func(context.Context) (string, error) { + return "browser-device-scope", nil + }, + mediaPoll: time.Millisecond, + mediaTimeout: time.Second, } } diff --git a/internal/canvas/canvas_test.go b/internal/canvas/canvas_test.go index 186c0ab..142cf4e 100644 --- a/internal/canvas/canvas_test.go +++ b/internal/canvas/canvas_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/common" ) @@ -124,6 +125,61 @@ func TestCreateTransportFailureExplainsAmbiguousOutcome(t *testing.T) { } } +func TestCreateCredentialUnavailableBeforeRequestIsProvenRetryable(t *testing.T) { + client := &fakeClient{send: func(context.Context, string, any, any) error { + return fmt.Errorf("authorizer rejected request: %w", internal_auth.ErrCredentialExpired) + }} + result, err := Create(context.Background(), CreateOptions{RequestID: "request-1"}, runnerWithClient(client)) + if result != nil || !errors.Is(err, internal_auth.ErrCredentialExpired) || + !strings.Contains(err.Error(), "was not sent") || strings.Contains(err.Error(), "outcome may be ambiguous") { + t.Fatalf("Create() result/error = %#v/%v, want typed pre-send credential failure", result, err) + } +} + +func TestCreatePollingCredentialExpiryPreservesAcceptedTypedError(t *testing.T) { + getThreadCalls := 0 + client := &fakeClient{send: func(_ context.Context, path string, _ any, out any) error { + switch path { + case CreatePath: + return decodeInto(out, `{"ret":"0","data":{"state":"creating","project_id":"100","thread_id":"thread-1","run_id":"run-1","canvas_asset_id":"200","web_url":"https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200"}}`) + case "/api/biz/v1/skill/get_thread": + getThreadCalls++ + return fmt.Errorf("get_thread auth: %w", internal_auth.ErrCredentialExpired) + default: + return fmt.Errorf("unexpected path %s", path) + } + }} + result, err := Create(context.Background(), CreateOptions{ + RequestID: "request-1", Wait: true, PollInterval: time.Millisecond, WaitTimeout: time.Second, + }, runnerWithClient(client)) + if result == nil || result.ProjectID != "100" || result.ThreadID != "thread-1" || getThreadCalls != 1 || + !errors.Is(err, internal_auth.ErrCredentialExpired) || !strings.Contains(err.Error(), "accepted canvas IDs") { + t.Fatalf("Create() result/error/calls = %#v/%v/%d", result, err, getThreadCalls) + } +} + +func TestResumeCreatePollingCredentialMissingPreservesAcceptedTypedError(t *testing.T) { + getThreadCalls := 0 + client := &fakeClient{send: func(_ context.Context, path string, _ any, _ any) error { + if path != "/api/biz/v1/skill/get_thread" { + return fmt.Errorf("unexpected path %s", path) + } + getThreadCalls++ + return fmt.Errorf("get_thread auth: %w", internal_auth.ErrCredentialNotFound) + }} + accepted := &CreateResult{ + RequestID: "request-1", State: StateCreating, ProjectID: "100", ThreadID: "thread-1", RunID: "run-1", + CanvasAssetID: "200", WebURL: "https://xyq.jianying.com/novel/detail/canvas?projectId=100&canvasId=200", + } + result, err := ResumeCreate(context.Background(), accepted, ResumeCreateOptions{ + PollInterval: time.Millisecond, WaitTimeout: time.Second, + }, runnerWithClient(client)) + if result == nil || result.ProjectID != accepted.ProjectID || getThreadCalls != 1 || + !errors.Is(err, internal_auth.ErrCredentialNotFound) || !strings.Contains(err.Error(), "accepted canvas IDs") { + t.Fatalf("ResumeCreate() result/error/calls = %#v/%v/%d", result, err, getThreadCalls) + } +} + func TestCreateWaitsForMachineArtifactWithoutV2(t *testing.T) { getThreadCalls := 0 client := &fakeClient{send: func(_ context.Context, path string, body any, out any) error { diff --git a/internal/canvas/create.go b/internal/canvas/create.go index ee5ef3d..93e65c7 100644 --- a/internal/canvas/create.go +++ b/internal/canvas/create.go @@ -9,6 +9,7 @@ import ( "strings" "time" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/common" ) @@ -93,6 +94,11 @@ func Create(ctx context.Context, opts CreateOptions, runner *common.Runner) (*Cr Base: base(), }, &envelope) if err != nil { + if IsCredentialUnavailable(err) { + // The authorizer rejected the request before http.Client.Do. This is + // the only create failure that is proven safe to retry after login. + return nil, fmt.Errorf("canvas create request was not sent because authentication is unavailable: %w", err) + } return nil, fmt.Errorf("canvas create request failed; outcome may be ambiguous, do not retry blindly: %w", err) } if err := envelope.validate("canvas create"); err != nil { @@ -127,6 +133,9 @@ func Create(ctx context.Context, opts CreateOptions, runner *common.Runner) (*Cr result.PollAttempts = attempts if waitErr != nil { result.Warning = waitErr.Error() + if IsCredentialUnavailable(waitErr) { + return result, acceptedCreationError(result, waitErr) + } var terminal *CreationTerminalError if errors.As(waitErr, &terminal) { result.State = "failed" @@ -154,12 +163,24 @@ func finalizeReadyCanvas(ctx context.Context, result *CreateResult, runner *comm if _, err := queryAssets(ctx, []string{result.CanvasAssetID}, true, runner); err != nil { result.State = StateCreating result.Warning = fmt.Sprintf("canvas overview is complete but the root asset is not queryable yet: %v", err) + if IsCredentialUnavailable(err) { + return result, acceptedCreationError(result, err) + } return result, nil } result.State = StateReady return result, nil } +// IsCredentialUnavailable identifies local credential state errors that the +// shared authorizer returns before issuing an HTTP request. errors.Is remains +// intact through every canvas/create wrapper so callers never infer safety +// from user-facing warning text. +func IsCredentialUnavailable(err error) bool { + return errors.Is(err, internal_auth.ErrCredentialNotFound) || + errors.Is(err, internal_auth.ErrCredentialExpired) +} + func acceptedCreationError(result *CreateResult, cause error) error { return fmt.Errorf( "%w; accepted canvas IDs: project_id=%s thread_id=%s run_id=%s canvas_asset_id=%s; do not create again blindly", diff --git a/internal/canvas/import_facade.go b/internal/canvas/import_facade.go index 254ad30..0e3119a 100644 --- a/internal/canvas/import_facade.go +++ b/internal/canvas/import_facade.go @@ -51,6 +51,9 @@ func ResumeCreate( result.PollAttempts += attempts if waitErr != nil { result.Warning = waitErr.Error() + if IsCredentialUnavailable(waitErr) { + return &result, acceptedCreationError(&result, waitErr) + } var terminal *CreationTerminalError if errors.As(waitErr, &terminal) { result.State = "failed" diff --git a/internal/canvasplan/canvasplan_test.go b/internal/canvasplan/canvasplan_test.go index 1764c25..bd56da1 100644 --- a/internal/canvasplan/canvasplan_test.go +++ b/internal/canvasplan/canvasplan_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/canvas" ) @@ -364,6 +365,96 @@ func TestExecutorNeverReplaysAmbiguousCreate(t *testing.T) { } } +func TestExecutorRetriesOnlyProvenPreSendCredentialFailure(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + api.createErr = fmt.Errorf("prepare canvas create: %w", internal_auth.ErrCredentialExpired) + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "pre-send-auth.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if !errors.Is(err, internal_auth.ErrCredentialExpired) || result == nil || result.State != StateInitialized { + t.Fatalf("first Execute() result=%#v error=%v, want initialized typed credential failure", result, err) + } + if api.createCalls != 1 || api.resumeCreateCalls != 0 { + t.Fatalf("first call create=%d resume=%d, want create=1 resume=0", api.createCalls, api.resumeCreateCalls) + } + journal := readJournal(t, journalPath) + if journal.State != StateInitialized || journal.Create != nil { + t.Fatalf("journal after pre-send failure = %#v, want initialized without accepted create", journal) + } + + api.createErr = nil + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil || result == nil || result.State != StateVerified { + t.Fatalf("second Execute() result=%#v error=%v, want verified", result, err) + } + if api.createCalls != 2 || api.resumeCreateCalls != 0 { + t.Fatalf("calls after safe retry create=%d resume=%d, want create=2 resume=0", api.createCalls, api.resumeCreateCalls) + } +} + +func TestExecutorNeverRecreatesAfterAcceptedCreateCredentialExpiry(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + api.createErr = fmt.Errorf("poll accepted canvas create: %w", internal_auth.ErrCredentialExpired) + api.createAcceptedOnError = true + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "accepted-create-auth.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if !errors.Is(err, internal_auth.ErrCredentialExpired) || result == nil || result.State != StateCreatePending { + t.Fatalf("first Execute() result=%#v error=%v, want pending typed credential failure", result, err) + } + if result.ProjectID == "" || api.createCalls != 1 || api.resumeCreateCalls != 0 { + t.Fatalf("accepted create result=%#v calls create=%d resume=%d", result, api.createCalls, api.resumeCreateCalls) + } + journal := readJournal(t, journalPath) + if journal.State != StateCreatePending || journal.Create == nil || journal.Create.ProjectID == "" { + t.Fatalf("journal did not preserve accepted IDs: %#v", journal) + } + + api.createErr = nil + api.createAcceptedOnError = false + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil || result == nil || result.State != StateVerified { + t.Fatalf("second Execute() result=%#v error=%v, want verified", result, err) + } + if api.createCalls != 1 || api.resumeCreateCalls != 1 { + t.Fatalf("accepted create was replayed: create=%d resume=%d", api.createCalls, api.resumeCreateCalls) + } +} + +func TestExecutorKeepsAcceptedCreatePendingAcrossResumeCredentialFailure(t *testing.T) { + plan, resolved := testPlanAndResolved() + api := newFakeCanvasAPI(len(plan.Nodes)) + api.createPending = true + executor := &Executor{api: api} + journalPath := filepath.Join(t.TempDir(), "resume-create-auth.json") + + result, err := executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil || result == nil || result.State != StateCreatePending { + t.Fatalf("first Execute() result=%#v error=%v, want pending", result, err) + } + api.resumeCreateErr = fmt.Errorf("resume accepted canvas create: %w", internal_auth.ErrCredentialNotFound) + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if !errors.Is(err, internal_auth.ErrCredentialNotFound) || result == nil || result.State != StateCreatePending { + t.Fatalf("second Execute() result=%#v error=%v, want pending typed credential failure", result, err) + } + if api.createCalls != 1 || api.resumeCreateCalls != 1 { + t.Fatalf("calls after resume failure create=%d resume=%d, want create=1 resume=1", api.createCalls, api.resumeCreateCalls) + } + + api.resumeCreateErr = nil + result, err = executor.Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if err != nil || result == nil || result.State != StateVerified { + t.Fatalf("third Execute() result=%#v error=%v, want verified", result, err) + } + if api.createCalls != 1 || api.resumeCreateCalls != 2 { + t.Fatalf("accepted create was replayed: create=%d resume=%d", api.createCalls, api.resumeCreateCalls) + } +} + func TestExecutorReportsQueryBackMismatchWithoutReplay(t *testing.T) { plan, resolved := testPlanAndResolved() api := newFakeCanvasAPI(len(plan.Nodes)) @@ -490,6 +581,8 @@ type fakeCanvasAPI struct { applyCalls int createPending bool createErr error + createAcceptedOnError bool + resumeCreateErr error getErr error getNil bool applyErr error @@ -506,6 +599,12 @@ func newFakeCanvasAPI(nodeCount int) *fakeCanvasAPI { func (api *fakeCanvasAPI) Create(context.Context, canvas.CreateOptions) (*canvas.CreateResult, error) { api.createCalls++ if api.createErr != nil { + if api.createAcceptedOnError { + return &canvas.CreateResult{ + RequestID: "request-1", State: canvas.StateCreating, ProjectID: "123", ThreadID: "thread-1", RunID: "run-1", + CanvasAssetID: "root-asset", WebURL: "/novel/detail/canvas?projectId=123", Warning: api.createErr.Error(), + }, api.createErr + } return nil, api.createErr } state := canvas.StateReady @@ -522,6 +621,12 @@ func (api *fakeCanvasAPI) Create(context.Context, canvas.CreateOptions) (*canvas func (api *fakeCanvasAPI) ResumeCreate(context.Context, *canvas.CreateResult, canvas.ResumeCreateOptions) (*canvas.CreateResult, error) { api.resumeCreateCalls++ + if api.resumeCreateErr != nil { + return &canvas.CreateResult{ + RequestID: "request-1", State: canvas.StateCreating, ProjectID: "123", ThreadID: "thread-1", RunID: "run-1", + CanvasAssetID: "root-asset", WebURL: "/novel/detail/canvas?projectId=123", Warning: api.resumeCreateErr.Error(), + }, api.resumeCreateErr + } api.createPending = false return &canvas.CreateResult{ RequestID: "request-1", State: canvas.StateReady, ProjectID: "123", ThreadID: "thread-1", RunID: "run-1", diff --git a/internal/canvasplan/executor.go b/internal/canvasplan/executor.go index 3ea7e82..497159e 100644 --- a/internal/canvasplan/executor.go +++ b/internal/canvasplan/executor.go @@ -236,6 +236,25 @@ func finishCreate(journalPath string, journal *Journal, created *canvas.CreateRe journal.Create = created } if createErr != nil { + if canvas.IsCredentialUnavailable(createErr) { + if created == nil { + // The shared authorizer proves that no HTTP request was issued. + // Roll back only this local marker so the import may authenticate + // and make its one allowed Create call. + return recordJournalError(journalPath, journal, StateInitialized, createErr) + } + // Create already returned durable IDs. Persist them as pending and + // let the next attempt call ResumeCreate only; never send Create again. + journal.State = StateCreatePending + journal.LastError = sanitizeJournalError(createErr.Error()) + if strings.TrimSpace(created.Warning) == "" { + created.Warning = journal.LastError + } + if err := saveJournal(journalPath, journal); err != nil { + return fmt.Errorf("%w; additionally failed to save CanvasPlan journal: %v", createErr, err) + } + return createErr + } state := StateCreateAmbiguous if created != nil { state = StateCreateFailed From 4a38233420feab277fb24730828a76504489a7c5 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 07:24:37 +0800 Subject: [PATCH 42/48] docs(auth): document browser login and CI override Co-authored-by: Codex <codex@openai.com> --- README.md | 16 ++++++++++------ skills/short-drama/SKILL.md | 8 ++++---- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d002217..135aa1e 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ Canvas beta 暴露个人漫剧画布的通用原子命令，不在服务端增 ```bash npx @pippit-dev/cli@beta install -export XYQ_ACCESS_KEY="<access-key>" +pippit-tool-cli login # 仅测试 PPE 时配置；生产环境不要设置 export PIPPIT_CLI_PPE_ENV="ppe_cli_canvas_ak" @@ -219,7 +219,9 @@ pippit-tool-cli canvas apply --project-id PROJECT_ID --file ./patch.json `canvas create/get/apply/upload` 的输出均为单行 JSON，所有资源 ID 保持字符串，便于脚本和 Agent 调用。`create` 的 `request_id` 当前用于追踪和恢复，不是跨服务崩溃窗口的严格幂等键；请求结果不明确时不要盲目重试。beta 的 `apply` 每次只接受一个 transaction（可以包含多个 patches），并严格检查该 transaction 的 ACK 和每个资产版本；当前服务端仍不保证跨资产 all-or-nothing，调用方应在写后执行 `get` 校验，并持久化自己的 operation journal。 -PPE 只影响 Pippit API 同源请求。CLI 不会把 Access Key、`x-tt-env`、`x-use-ppe` 或 `x-schedule-vdc` 转发给第三方绝对 URL；`--ppe-env` 的优先级高于 `PIPPIT_CLI_PPE_ENV`，二者都未提供时访问生产环境。 +`login` 会打开小云雀网页完成授权，并把本机设备专属凭证保存到系统安全凭证库；不会在终端显示 Access Key。后续原生 CLI 命令会自动读取该凭证。`XYQ_ACCESS_KEY` 仅作为 CI/Agent 的显式覆盖保留，且优先级高于网页登录凭证；配置错误时不会静默回退到个人登录。 + +PPE 只影响登录完成后的 Pippit API 同源业务请求，不改变登录账号或凭证。CLI 不会把 Access Key、`x-tt-env`、`x-use-ppe` 或 `x-schedule-vdc` 转发给第三方绝对 URL；`--ppe-env` 的优先级高于 `PIPPIT_CLI_PPE_ENV`，二者都未提供时访问生产环境。同一个有效登录凭证可用于生产环境和 PPE。 ### 一键导入 LibTV 画布 @@ -235,11 +237,11 @@ CLI 会在交互终端中显示彩色向导：使用 ↑/↓ 移动、Enter 确 给定链接后，CLI 会通过官方 LibTV CLI 完成网页授权与草稿/素材导出，再依次调用通用的 `upload`、`create`、内部 ID 分配、单 transaction `apply` 和 `get` 全量校验。 -生产环境使用时删除 `--ppe-env ppe_cli_canvas_ak`。当前登录能力仍沿用既有 Access Key 配置；CLI 自动保存 AK 的 `login` 流程会单独交付。 +生产环境使用时删除 `--ppe-env ppe_cli_canvas_ak`。交互式导入会在下载 LibTV 项目和素材之前依次校验小云雀与 LibTV 登录；小云雀未登录或凭证失效时会直接打开浏览器授权，成功后在同一进程继续，不再要求粘贴 Access Key。 首次运行时，若本机没有 LibTV CLI，导入器只会从 LibTV 官方静态域下载固定版本 1.1.3 的对应平台 ZIP，并同时校验 ZIP 和可执行文件的内置 SHA-256；不会执行远程安装脚本。若官方 LibTV CLI 尚未登录，它会打开 `libtv login web --open` 的官方网页授权流程，导入器本身不读取浏览器 Cookie 或 LibTV credential 文件。 -`--accept-degradations` 表示接受计划中明确列出的不可移植节点。例如没有生成结果的图片/视频节点会保留为空占位，LibTV 私有 `video-clip` 会降级成空的 Pippit video-composite。交互式导入会就地询问是否接受；非交互调用未传该参数时，CLI 会在任何 Pippit 写入前停止。 +`--accept-degradations` 表示接受计划中明确列出的不可移植节点。例如没有生成结果的图片/视频节点会保留为空占位，LibTV 私有 `video-clip` 会降级成空的 Pippit video-composite。交互式导入会显示中文 warning 后自动继续；非交互调用未传该参数时，CLI 会在任何 Pippit 写入前停止。 导入状态会写入权限为 `0600` 的本地 journal。素材上传、画布创建或 transaction 结果不明确时，CLI 会保留已获得的持久 ID 并拒绝盲目重复写入；重复执行同一条命令会优先 query-back 恢复。只有 root 和所有伴生资产逐一通过 canonical hash 校验后，命令才返回 `state=verified` 并执行 `--open`。 @@ -261,7 +263,7 @@ pippit-tool-cli libtv plan \ ```bash npx @pippit-dev/cli@latest install -export XYQ_ACCESS_KEY="<access-key>" +pippit-tool-cli login pippit-tool-cli --version pippit-tool-cli short-drama +submit-run --message "写一个赛博朋克短剧开头" pippit-tool-cli short-drama +upload-file --path ./reference.doc @@ -389,4 +391,6 @@ pippit-tool-cli query-result \ ## 鉴权 -`short-drama +submit-run`、`get-thread`、`list-thread-file`、`short-drama +upload-file` 以及 `xyq-skill` Python 脚本都使用 `Authorization: Bearer <XYQ_ACCESS_KEY>` 鉴权。OAuth 命令代码仍保留在仓库中，但短剧运行时请求不使用 OAuth。 +原生 CLI 命令通过 `pippit-tool-cli login` 获取并安全保存的设备专属凭证鉴权。可用 `pippit-tool-cli status` 查看状态、`pippit-tool-cli logout` 清除本机网页登录凭证；这些命令都不会输出 Access Key。CI/Agent 可继续显式设置 `XYQ_ACCESS_KEY`，它会覆盖本机网页登录凭证。 + +`skills/xyq-nest-skill/scripts` 下的独立 Python 脚本尚未接入原生 CLI 的安全凭证库，当前仍需要 `XYQ_ACCESS_KEY`；不要把这一限制误解为 `pippit-tool-cli` 原生命令仍需手工配置 AK。 diff --git a/skills/short-drama/SKILL.md b/skills/short-drama/SKILL.md index 957601b..597e927 100644 --- a/skills/short-drama/SKILL.md +++ b/skills/short-drama/SKILL.md @@ -86,14 +86,14 @@ metadata: npx @pippit-dev/cli@latest install ``` -部分功能需要先配置 `XYQ_ACCESS_KEY`。缺失时 CLI 会直接提示用户先创建 Access Key，此时请等待用户给与Access Key后再继续运行。 - -Access Key 创建地址：https://xyq.jianying.com/home?tab_name=home +首次使用原生 CLI 时运行网页登录；CLI 会自动申请或复用本机专属凭证，并保存到系统安全凭证库，不要求用户复制 Access Key： ```bash -export XYQ_ACCESS_KEY="<access-key>" +pippit-tool-cli login ``` +`XYQ_ACCESS_KEY` 仅保留给 CI、Agent 等非交互环境作为显式覆盖。若该环境变量已设置但无效，CLI 不会静默改用个人网页登录凭证；应先修正或取消该环境变量。 + ## 小云雀界面打开契约 `+submit-run` 返回 `web_thread_link` 后，用户侧 Agent 必须优先把小云雀短剧 WebUI 打开给用户，而不是只展示链接。 From 6393bbce7eaadf2400b0c3363ec99dde6676dbb9 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 08:40:10 +0800 Subject: [PATCH 43/48] fix(auth): initialize the secure file fallback Co-authored-by: Codex <codex@openai.com> --- internal/auth/auth_test.go | 34 ++++++++++++++++++++++++++++++++++ internal/auth/store.go | 5 ++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 3b435a4..06ee9ee 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -581,6 +581,40 @@ func TestResilientStoreFallsBackOnlyToSecureStore(t *testing.T) { } } +func TestManagerFreshIdentityUsesEmptyFallbackWhenKeyringUnavailable(t *testing.T) { + primary := &memoryCredentialStore{loadErr: ErrSecureStore, saveErr: ErrSecureStore} + fallback := &memoryCredentialStore{} + store := &resilientCredentialStore{primary: primary, fallback: fallback} + manager := NewManager( + config.Load(), + WithCredentialStore(store), + withRandomReaderForTest(bytes.NewReader(bytes.Repeat([]byte{0x2a}, deviceIDBytes))), + ) + + identity, err := manager.ensureIdentity(context.Background()) + if err != nil { + t.Fatalf("ensureIdentity() error = %v", err) + } + if identity.AccessKey != "" || !validDeviceID(identity.DeviceID) { + t.Fatalf("fresh identity = %#v", credentialWithoutSecret(identity)) + } + fallback.mu.Lock() + stored := cloneCredential(fallback.credential) + fallbackLoads, fallbackSaves := fallback.loads, fallback.saves + fallback.mu.Unlock() + if stored == nil || stored.DeviceID != identity.DeviceID || fallbackLoads != 1 || fallbackSaves != 1 { + t.Fatalf("fallback identity/loads/saves = %#v/%d/%d", credentialWithoutSecret(stored), fallbackLoads, fallbackSaves) + } + if primary.loads != 1 || primary.saves != 1 { + t.Fatalf("primary loads/saves = %d/%d, want 1/1", primary.loads, primary.saves) + } + + again, err := manager.ensureIdentity(context.Background()) + if err != nil || again.DeviceID != identity.DeviceID || fallback.loads != 1 { + t.Fatalf("cached ensureIdentity() = %#v/%v, fallback loads=%d", credentialWithoutSecret(again), err, fallback.loads) + } +} + func TestResilientStoreDoesNotMaskCorruptPrimaryOrCleanupFailure(t *testing.T) { identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{4}, deviceIDBytes))) corrupt := errors.New("corrupt primary credential") diff --git a/internal/auth/store.go b/internal/auth/store.go index 970310b..81d4d47 100644 --- a/internal/auth/store.go +++ b/internal/auth/store.go @@ -148,7 +148,10 @@ func (s *resilientCredentialStore) Load(ctx context.Context) (*Credential, error if fallbackErr == nil { return credential, nil } - if errors.Is(primaryErr, ErrCredentialNotFound) && errors.Is(fallbackErr, ErrCredentialNotFound) { + if errors.Is(fallbackErr, ErrCredentialNotFound) { + // An available, empty fallback is the active store when the primary + // keyring is unavailable. Report a fresh login state so Manager can + // create the device identity and Save can persist it to that fallback. return nil, ErrCredentialNotFound } if !errors.Is(fallbackErr, ErrCredentialNotFound) { From 840d7a3a8f94fc0bf3c5b6d92b91316ce68daf9a Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 08:42:53 +0800 Subject: [PATCH 44/48] fix(canvas): retry only proven authentication rejections Co-authored-by: Codex <codex@openai.com> --- cmd/canvas/import_auth.go | 13 +++++- cmd/canvas/import_auth_flow_test.go | 58 +++++++++++++++++++++++++- cmd/canvas/import_media.go | 2 +- cmd/canvas/import_test.go | 38 +++++++++++++++++ internal/auth/types.go | 1 + internal/canvas/canvas_test.go | 47 ++++++++++++++++++++- internal/canvas/create.go | 10 +++-- internal/canvas/types.go | 9 +++- internal/canvasplan/canvasplan_test.go | 36 ++++++++++++++++ internal/common/client.go | 9 +++- 10 files changed, 213 insertions(+), 10 deletions(-) diff --git a/cmd/canvas/import_auth.go b/cmd/canvas/import_auth.go index 027f78e..8161832 100644 --- a/cmd/canvas/import_auth.go +++ b/cmd/canvas/import_auth.go @@ -313,7 +313,8 @@ func isCanvasImportPippitAuthFailure(err error) bool { } if errors.Is(err, errCanvasImportReauthenticationRequired) || errors.Is(err, internal_auth.ErrCredentialNotFound) || - errors.Is(err, internal_auth.ErrCredentialExpired) { + errors.Is(err, internal_auth.ErrCredentialExpired) || + errors.Is(err, internal_auth.ErrCredentialRejected) { return true } message := strings.ToLower(err.Error()) @@ -328,3 +329,13 @@ func isCanvasImportPippitAuthFailure(err error) bool { } return false } + +// isProvenPrewriteCredentialFailure is intentionally stricter than the +// user-facing detector above. Only structured credential failures prove that +// a protected write was rejected before execution; message substrings are not +// sufficient evidence for clearing a durable upload-requested marker. +func isProvenPrewriteCredentialFailure(err error) bool { + return errors.Is(err, internal_auth.ErrCredentialNotFound) || + errors.Is(err, internal_auth.ErrCredentialExpired) || + errors.Is(err, internal_auth.ErrCredentialRejected) +} diff --git a/cmd/canvas/import_auth_flow_test.go b/cmd/canvas/import_auth_flow_test.go index 3712899..6841f85 100644 --- a/cmd/canvas/import_auth_flow_test.go +++ b/cmd/canvas/import_auth_flow_test.go @@ -3,9 +3,13 @@ package canvas import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" + "net/http" + "net/http/httptest" + "os" "path/filepath" "strings" "testing" @@ -14,6 +18,7 @@ import ( internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" "github.com/Pippit-dev/pippit-cli/internal/canvasplan" + "github.com/Pippit-dev/pippit-cli/internal/common" ) type trackingImportAuthAPI struct { @@ -121,7 +126,7 @@ func (api *expiringImportMediaAPI) Upload( ) (*canvascore.UploadResult, error) { api.uploads++ if api.uploads == 1 && !api.uploadSucceeds { - return nil, errors.New("HTTP 401") + return nil, fmt.Errorf("HTTP 401: %w", internal_auth.ErrCredentialRejected) } return &canvascore.UploadResult{ State: canvascore.StateReady, AssetID: "asset-after-reauth", PippitAssetID: "pippit-after-reauth", @@ -550,6 +555,57 @@ func TestCanvasImportReauthorizesWhileWaitingForAcceptedCreate(t *testing.T) { } } +func TestCanvasImportRealInitialCreateRejectionReauthenticatesFromInitialized(t *testing.T) { + temp := t.TempDir() + plan, mediaBytes := testImportPlan(t, false) + exporter := &fakeImportExporter{plan: plan, mediaBytes: mediaBytes} + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requests++ + if request.URL.Path != canvascore.CreatePath { + http.NotFound(writer, request) + return + } + if request.Header.Get("Authorization") != "Bearer expired-ak" { + t.Errorf("Authorization = %q", request.Header.Get("Authorization")) + } + http.Error(writer, "expired", http.StatusUnauthorized) + })) + defer server.Close() + client := common.NewHTTPClient(server.URL, time.Second, common.NewAccessKeyAuthorizer("expired-ak")) + runner := common.NewRunner(nil, client) + pippit := &trackingImportAuthAPI{ + key: "expired-ak", + loginErrors: []error{errors.New("stop after observing browser reauthentication")}, + } + deps := testImportDependencies(temp, exporter, &fakeImportMediaAPI{}, runnerImportExecutor{executor: canvasplan.NewExecutor(runner)}) + deps.pippitAuth = pippit + journalPath := filepath.Join(temp, "real-create-auth.journal.json") + var stderr bytes.Buffer + prompts := newImportPromptSessionWithTUI(context.Background(), strings.NewReader(""), &stderr, false) + + result, err := runCanvasImport(context.Background(), importOptions{ + Provider: "libtv", SourceURL: testLibTVURL, JournalPath: journalPath, JournalExplicit: true, + }, deps, &stderr, prompts) + if err == nil || !strings.Contains(err.Error(), "网页重新授权失败") { + t.Fatalf("runCanvasImport() result/error = %#v/%v, want browser reauthentication attempt", result, err) + } + if result == nil || result.State != canvasplan.StateInitialized || pippit.logins != 1 || requests != 1 { + t.Fatalf("result/logins/requests = %#v/%d/%d, want initialized/1/1", result, pippit.logins, requests) + } + payload, readErr := os.ReadFile(journalPath) + if readErr != nil { + t.Fatal(readErr) + } + journal := &canvasplan.Journal{} + if err := json.Unmarshal(payload, journal); err != nil || journal.State != canvasplan.StateInitialized || journal.Create != nil { + t.Fatalf("journal/error = %#v/%v, want durable initialized state", journal, err) + } + if !strings.Contains(stderr.String(), "断点已保存") { + t.Fatalf("stderr missing safe retry guidance: %q", stderr.String()) + } +} + func TestCanvasImportReauthorizesAmbiguousApplyThenQueriesWithoutReplay(t *testing.T) { temp := t.TempDir() plan, mediaBytes := testImportPlan(t, false) diff --git a/cmd/canvas/import_media.go b/cmd/canvas/import_media.go index 80d3125..ff0cc32 100644 --- a/cmd/canvas/import_media.go +++ b/cmd/canvas/import_media.go @@ -339,7 +339,7 @@ func resolveImportMedia( entries[media.LogicalID] = &entry reportImportMediaProgress(stderr, index, len(opts.Media), "uploading", media) uploaded, uploadErr := api.Upload(ctx, media) - if uploadErr != nil && isCanvasImportPippitAuthFailure(uploadErr) { + if uploadErr != nil && isProvenPrewriteCredentialFailure(uploadErr) { if checkpointErr := removeAndSaveMediaEntry(opts.CheckpointPath, checkpoint, media.LogicalID); checkpointErr != nil { return canvasplan.ResolvedMediaSet{}, fmt.Errorf( "发送素材 %q 的上传请求前，小云雀授权失败，且无法清理 upload-requested 断点记录；请勿直接重试：%w", diff --git a/cmd/canvas/import_test.go b/cmd/canvas/import_test.go index 3e5b31f..18f6540 100644 --- a/cmd/canvas/import_test.go +++ b/cmd/canvas/import_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" canvascore "github.com/Pippit-dev/pippit-cli/internal/canvas" "github.com/Pippit-dev/pippit-cli/internal/canvasplan" ) @@ -903,6 +904,43 @@ func TestImportCommandBlocksUnknownUploadOutcome(t *testing.T) { } } +func TestMediaUploadTextHeuristicCannotClearAmbiguousMarker(t *testing.T) { + opts := testMediaResolutionOptions(t) + first := &fakeImportMediaAPI{uploadErr: errors.New("HTTP 500: upstream diagnostic mentioned unauthorized")} + _, err := resolveImportMedia(context.Background(), opts, first, io.Discard) + if err == nil || !strings.Contains(err.Error(), "blocked") { + t.Fatalf("first resolve error = %v, want blocked unknown outcome", err) + } + checkpoint := readTestMediaCheckpoint(t, opts.CheckpointPath) + if len(checkpoint.Entries) != 1 || checkpoint.Entries[0].Status != mediaStatusBlocked { + t.Fatalf("checkpoint entries = %#v, want durable blocked marker", checkpoint.Entries) + } + + second := &fakeImportMediaAPI{} + _, err = resolveImportMedia(context.Background(), opts, second, io.Discard) + if err == nil || !strings.Contains(err.Error(), "blocked") || second.uploads != 0 { + t.Fatalf("second resolve error/uploads = %v/%d, want no blind re-upload", err, second.uploads) + } +} + +func TestMediaUploadTypedCredentialRejectionClearsMarkerForRetry(t *testing.T) { + opts := testMediaResolutionOptions(t) + first := &fakeImportMediaAPI{uploadErr: fmt.Errorf("upload rejected: %w", internal_auth.ErrCredentialRejected)} + _, err := resolveImportMedia(context.Background(), opts, first, io.Discard) + if !errors.Is(err, errCanvasImportReauthenticationRequired) || first.uploads != 1 { + t.Fatalf("first resolve error/uploads = %v/%d, want structured reauthentication", err, first.uploads) + } + if checkpoint := readTestMediaCheckpoint(t, opts.CheckpointPath); len(checkpoint.Entries) != 0 { + t.Fatalf("checkpoint entries = %#v, want rejected prewrite marker removed", checkpoint.Entries) + } + + second := &fakeImportMediaAPI{} + resolved, err := resolveImportMedia(context.Background(), opts, second, io.Discard) + if err != nil || second.uploads != 1 || len(resolved.Media) != 1 { + t.Fatalf("second resolve result/error/uploads = %#v/%v/%d", resolved, err, second.uploads) + } +} + func TestMediaCheckpointBlocksResumeAfterUploadCrashWindow(t *testing.T) { opts := testMediaResolutionOptions(t) crashing := &panickingImportMediaAPI{} diff --git a/internal/auth/types.go b/internal/auth/types.go index 8786e3f..d84518d 100644 --- a/internal/auth/types.go +++ b/internal/auth/types.go @@ -26,6 +26,7 @@ const ( var ( ErrCredentialNotFound = errors.New("未找到本机小云雀 CLI 登录凭证") ErrCredentialExpired = errors.New("本机小云雀 CLI 登录凭证已过期") + ErrCredentialRejected = errors.New("小云雀拒绝了当前 CLI 登录凭证") ErrSecureStore = errors.New("安全凭证存储不可用") ErrCredentialAccountMismatch = errors.New("网页授权账号与当前任务账号不一致") ErrRemoteRevokeUnsupported = errors.New("当前版本不支持在 CLI 中安全撤销远程 Access Key") diff --git a/internal/canvas/canvas_test.go b/internal/canvas/canvas_test.go index 142cf4e..4482dad 100644 --- a/internal/canvas/canvas_test.go +++ b/internal/canvas/canvas_test.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" @@ -131,11 +133,54 @@ func TestCreateCredentialUnavailableBeforeRequestIsProvenRetryable(t *testing.T) }} result, err := Create(context.Background(), CreateOptions{RequestID: "request-1"}, runnerWithClient(client)) if result != nil || !errors.Is(err, internal_auth.ErrCredentialExpired) || - !strings.Contains(err.Error(), "was not sent") || strings.Contains(err.Error(), "outcome may be ambiguous") { + !strings.Contains(err.Error(), "not accepted") || strings.Contains(err.Error(), "outcome may be ambiguous") { t.Fatalf("Create() result/error = %#v/%v, want typed pre-send credential failure", result, err) } } +func TestCreateRealHTTPAuthenticationRejectionIsTypedAndNotAmbiguous(t *testing.T) { + tests := []struct { + name string + status int + body string + wantRejected bool + }{ + {name: "HTTP 401", status: http.StatusUnauthorized, body: `{"message":"expired"}`, wantRejected: true}, + {name: "HTTP 403", status: http.StatusForbidden, body: `{"message":"forbidden"}`, wantRejected: true}, + {name: "business ret 1015", status: http.StatusOK, body: `{"ret":"1015","errmsg":"invalid access key","log_id":"log-auth","data":{}}`, wantRejected: true}, + {name: "HTTP 500", status: http.StatusInternalServerError, body: `{"message":"temporary"}`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != CreatePath { + http.NotFound(writer, request) + return + } + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(test.status) + _, _ = io.WriteString(writer, test.body) + })) + defer server.Close() + client := common.NewHTTPClient(server.URL, time.Second, common.NewAccessKeyAuthorizer("test-ak")) + + result, err := Create(context.Background(), CreateOptions{RequestID: "request-real-auth"}, common.NewRunner(nil, client)) + if result != nil || err == nil { + t.Fatalf("Create() result/error = %#v/%v, want failure", result, err) + } + if got := errors.Is(err, internal_auth.ErrCredentialRejected); got != test.wantRejected { + t.Fatalf("errors.Is(ErrCredentialRejected) = %v, error=%v", got, err) + } + if test.wantRejected && strings.Contains(err.Error(), "outcome may be ambiguous") { + t.Fatalf("explicit rejection was marked ambiguous: %v", err) + } + if !test.wantRejected && !strings.Contains(err.Error(), "outcome may be ambiguous") { + t.Fatalf("non-auth server failure lost ambiguous safety state: %v", err) + } + }) + } +} + func TestCreatePollingCredentialExpiryPreservesAcceptedTypedError(t *testing.T) { getThreadCalls := 0 client := &fakeClient{send: func(_ context.Context, path string, _ any, out any) error { diff --git a/internal/canvas/create.go b/internal/canvas/create.go index 93e65c7..a77fd31 100644 --- a/internal/canvas/create.go +++ b/internal/canvas/create.go @@ -95,9 +95,10 @@ func Create(ctx context.Context, opts CreateOptions, runner *common.Runner) (*Cr }, &envelope) if err != nil { if IsCredentialUnavailable(err) { - // The authorizer rejected the request before http.Client.Do. This is - // the only create failure that is proven safe to retry after login. - return nil, fmt.Errorf("canvas create request was not sent because authentication is unavailable: %w", err) + // The authorizer rejected before http.Client.Do, or the service + // explicitly rejected authentication with 401/403. Both prove that + // no create write was accepted and are safe to retry after login. + return nil, fmt.Errorf("canvas create was not accepted because authentication is unavailable or rejected: %w", err) } return nil, fmt.Errorf("canvas create request failed; outcome may be ambiguous, do not retry blindly: %w", err) } @@ -178,7 +179,8 @@ func finalizeReadyCanvas(ctx context.Context, result *CreateResult, runner *comm // from user-facing warning text. func IsCredentialUnavailable(err error) bool { return errors.Is(err, internal_auth.ErrCredentialNotFound) || - errors.Is(err, internal_auth.ErrCredentialExpired) + errors.Is(err, internal_auth.ErrCredentialExpired) || + errors.Is(err, internal_auth.ErrCredentialRejected) } func acceptedCreationError(result *CreateResult, cause error) error { diff --git a/internal/canvas/types.go b/internal/canvas/types.go index 6ffb5a3..7a93923 100644 --- a/internal/canvas/types.go +++ b/internal/canvas/types.go @@ -6,6 +6,7 @@ import ( "net/url" "strings" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/common" ) @@ -40,10 +41,16 @@ func (r responseEnvelope) validate(operation string) error { if message == "" { message = "unknown error" } - return common.NewLogIDError( + responseErr := common.NewLogIDError( fmt.Sprintf("%s failed: ret=%q errmsg=%s", operation, r.Ret, message), r.LogID, ) + if r.Ret == "1015" { + // ret=1015 is the service's explicit credential rejection. Unlike a + // transport failure or 5xx, it proves the business write was rejected. + return fmt.Errorf("%w: %w", internal_auth.ErrCredentialRejected, responseErr) + } + return responseErr } func base() map[string]any { diff --git a/internal/canvasplan/canvasplan_test.go b/internal/canvasplan/canvasplan_test.go index bd56da1..4410ad9 100644 --- a/internal/canvasplan/canvasplan_test.go +++ b/internal/canvasplan/canvasplan_test.go @@ -7,14 +7,18 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "net/http/httptest" "os" "path/filepath" "reflect" "strings" "testing" + "time" internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/canvas" + "github.com/Pippit-dev/pippit-cli/internal/common" ) func TestMaterializeCanonicalDocumentWithoutTransientMediaLocations(t *testing.T) { @@ -394,6 +398,38 @@ func TestExecutorRetriesOnlyProvenPreSendCredentialFailure(t *testing.T) { } } +func TestExecutorRealCreateAuthRejectionReturnsToInitialized(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requests++ + if request.URL.Path != canvas.CreatePath { + http.NotFound(writer, request) + return + } + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]any{ + "ret": "1015", "errmsg": "invalid access key", "log_id": "log-auth", "data": map[string]any{}, + }) + })) + defer server.Close() + client := common.NewHTTPClient(server.URL, time.Second, common.NewAccessKeyAuthorizer("expired-ak")) + runner := common.NewRunner(nil, client) + plan, resolved := testPlanAndResolved() + journalPath := filepath.Join(t.TempDir(), "real-auth-rejection.json") + + result, err := NewExecutor(runner).Execute(context.Background(), plan, resolved, ExecuteOptions{JournalPath: journalPath}) + if !errors.Is(err, internal_auth.ErrCredentialRejected) || result == nil || result.State != StateInitialized { + t.Fatalf("Execute() result/error = %#v/%v, want initialized typed auth rejection", result, err) + } + if requests != 1 { + t.Fatalf("HTTP requests = %d, want only the rejected Create", requests) + } + journal := readJournal(t, journalPath) + if journal.State != StateInitialized || journal.Create != nil { + t.Fatalf("journal after rejected Create = %#v", journal) + } +} + func TestExecutorNeverRecreatesAfterAcceptedCreateCredentialExpiry(t *testing.T) { plan, resolved := testPlanAndResolved() api := newFakeCanvasAPI(len(plan.Nodes)) diff --git a/internal/common/client.go b/internal/common/client.go index 65b686c..1444a55 100644 --- a/internal/common/client.go +++ b/internal/common/client.go @@ -15,6 +15,7 @@ import ( "strings" "time" + internal_auth "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/config" "github.com/bytedance/sonic" ) @@ -367,7 +368,13 @@ func (c *httpClient) do(req *http.Request, out any) error { if msg == "" { msg = http.StatusText(resp.StatusCode) } - return fmt.Errorf("%s %s 返回 HTTP %d: %s", req.Method, req.URL.String(), resp.StatusCode, msg) + responseErr := fmt.Errorf("%s %s 返回 HTTP %d: %s", req.Method, req.URL.String(), resp.StatusCode, msg) + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + // An explicit authentication/authorization rejection proves that + // the protected operation was not accepted for execution. + return fmt.Errorf("%w: %w", internal_auth.ErrCredentialRejected, responseErr) + } + return responseErr } if out == nil || len(data) == 0 { return nil From 3d6b25b3e07232f676853fc41cbc97c57b644e1a Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 08:44:07 +0800 Subject: [PATCH 45/48] fix(update): keep CLI credentials out of child processes Co-authored-by: Codex <codex@openai.com> --- cmd/update/update.go | 39 +++++++++++++++--- cmd/update/update_test.go | 84 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 5 deletions(-) diff --git a/cmd/update/update.go b/cmd/update/update.go index 9432290..00afe31 100644 --- a/cmd/update/update.go +++ b/cmd/update/update.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/Pippit-dev/pippit-cli/internal/auth" "github.com/Pippit-dev/pippit-cli/internal/version" "github.com/spf13/cobra" ) @@ -220,20 +221,48 @@ func runInherit(stderr io.Writer, name string, args ...string) error { func runInheritEnv(stderr io.Writer, env []string, name string, args ...string) error { cmd := command(name, args...) - if len(env) > 0 { - cmd.Env = append(os.Environ(), env...) - } + cmd.Env = sanitizedUpdateEnv(os.Environ(), env) cmd.Stdout = stderr cmd.Stderr = stderr return cmd.Run() } +func sanitizedUpdateEnv(environment, overrides []string) []string { + merged := overlayEnvironment(environment, overrides) + return auth.SanitizedBrowserEnv(merged) +} + +func overlayEnvironment(environment, overrides []string) []string { + result := make([]string, 0, len(environment)+len(overrides)) + indexes := make(map[string]int, len(environment)+len(overrides)) + for _, entry := range append(append([]string(nil), environment...), overrides...) { + name, _, found := strings.Cut(entry, "=") + if !found || strings.TrimSpace(name) == "" { + continue + } + key := name + if runtime.GOOS == "windows" { + key = strings.ToUpper(key) + } + if index, exists := indexes[key]; exists { + result[index] = entry + continue + } + indexes[key] = len(result) + result = append(result, entry) + } + return result +} func command(name string, args ...string) *exec.Cmd { + var cmd *exec.Cmd if runtime.GOOS == "windows" { cmdArgs := append([]string{"/c", name}, args...) - return exec.Command("cmd.exe", cmdArgs...) + cmd = exec.Command("cmd.exe", cmdArgs...) + } else { + cmd = exec.Command(name, args...) } - return exec.Command(name, args...) + cmd.Env = sanitizedUpdateEnv(os.Environ(), nil) + return cmd } func prepareSelfReplace() (func(), error) { diff --git a/cmd/update/update_test.go b/cmd/update/update_test.go index 41c357c..9bb99fa 100644 --- a/cmd/update/update_test.go +++ b/cmd/update/update_test.go @@ -172,3 +172,87 @@ func TestDefaultInstallPackageFollowsCurrentReleaseChannel(t *testing.T) { } } } + +func TestRunInheritEnvSanitizesPippitCredentials(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test uses a POSIX shell script to capture the child environment") + } + + binDir := t.TempDir() + capturePath := filepath.Join(t.TempDir(), "environment.txt") + commandPath := filepath.Join(binDir, "capture-update-environment") + script := "#!/bin/sh\n/usr/bin/env > \"$CAPTURE_ENV\"\n" + if err := os.WriteFile(commandPath, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + + basePath := binDir + string(os.PathListSeparator) + "/base/path" + overriddenPath := binDir + string(os.PathListSeparator) + "/overridden/path" + t.Setenv("PATH", basePath) + t.Setenv("CAPTURE_ENV", capturePath) + t.Setenv("SAFE_INHERITED", "kept") + t.Setenv("XYQ_ACCESS_KEY", "xyq-secret") + t.Setenv("PIPPIT_ACCESS_KEY", "pippit-secret") + t.Setenv("PIPPIT_AK", "legacy-secret") + t.Setenv("PIPPIT_CLI_TOKEN", "pippit-token") + t.Setenv("XYQ_CLIENT_SECRET", "xyq-client-secret") + t.Setenv("NPM_TOKEN", "npm-secret") + t.Setenv("NODE_AUTH_TOKEN", "registry-secret") + + var stderr bytes.Buffer + err := runInheritEnv(&stderr, []string{ + "PATH=" + overriddenPath, + "SAFE_INHERITED=overridden", + "SAFE_EXPLICIT=kept", + "PIPPIT_CLI_SKIP_SKILLS=1", + "XYQ_ACCESS_KEY=override-must-not-leak", + "PIPPIT_OVERRIDE_SECRET=override-must-not-leak", + "REGISTRY_TOKEN=explicit-registry-secret", + }, "capture-update-environment") + if err != nil { + t.Fatalf("runInheritEnv() error = %v, stderr = %s", err, stderr.String()) + } + + captured, err := os.ReadFile(capturePath) + if err != nil { + t.Fatal(err) + } + got := parseEnvironment(string(captured)) + for _, forbidden := range []string{ + "XYQ_ACCESS_KEY", + "PIPPIT_ACCESS_KEY", + "PIPPIT_AK", + "PIPPIT_CLI_TOKEN", + "XYQ_CLIENT_SECRET", + "PIPPIT_OVERRIDE_SECRET", + } { + if value, exists := got[forbidden]; exists { + t.Fatalf("child environment retained %s=%q", forbidden, value) + } + } + for name, want := range map[string]string{ + "PATH": overriddenPath, + "CAPTURE_ENV": capturePath, + "SAFE_INHERITED": "overridden", + "SAFE_EXPLICIT": "kept", + "PIPPIT_CLI_SKIP_SKILLS": "1", + "NPM_TOKEN": "npm-secret", + "NODE_AUTH_TOKEN": "registry-secret", + "REGISTRY_TOKEN": "explicit-registry-secret", + } { + if value := got[name]; value != want { + t.Fatalf("child environment %s = %q, want %q", name, value, want) + } + } +} + +func parseEnvironment(environment string) map[string]string { + result := make(map[string]string) + for _, entry := range strings.Split(environment, "\n") { + name, value, found := strings.Cut(entry, "=") + if found { + result[name] = value + } + } + return result +} From c49bf19866ac27aa9f471add0388956f74604b01 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 12:35:31 +0800 Subject: [PATCH 46/48] refactor(auth): receive managed keys from web Co-authored-by: Codex <codex@openai.com> --- internal/auth/api.go | 299 -------------------------------------- internal/auth/identity.go | 43 +++--- internal/auth/loopback.go | 115 +++++++++++---- internal/auth/manager.go | 78 +++++----- internal/auth/store.go | 20 +-- internal/auth/types.go | 29 ++-- 6 files changed, 173 insertions(+), 411 deletions(-) delete mode 100644 internal/auth/api.go diff --git a/internal/auth/api.go b/internal/auth/api.go deleted file mode 100644 index a432657..0000000 --- a/internal/auth/api.go +++ /dev/null @@ -1,299 +0,0 @@ -package auth - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/http/cookiejar" - "net/url" - "strconv" - "strings" -) - -const ( - maxAPIResponseBytes = 1 << 20 - loginGrantScope = "xyq_novel_cli_login" -) - -type apiEnvelope[T any] struct { - Ret json.RawMessage `json:"ret"` - Errmsg string `json:"errmsg"` - Data T `json:"data"` -} - -type apiResponseError struct { - operation string - httpStatus int - ret string -} - -func (err *apiResponseError) Error() string { - if err.httpStatus != 0 { - return fmt.Sprintf("%s失败（HTTP %d）", err.operation, err.httpStatus) - } - return fmt.Sprintf("%s失败（服务端错误码 %s）", err.operation, err.ret) -} - -type exchangeData struct { - UID string `json:"uid"` - Scope string `json:"scope"` -} - -type queryAccessKeyData struct { - AccessTokens []accessToken `json:"access_token_list"` -} - -type accessToken struct { - ID string `json:"ak_id"` - Token string `json:"token"` - ExpiredAt int64 `json:"expired_at"` - Name string `json:"token_name"` - Status string `json:"token_status"` -} - -type generateAccessKeyData struct { - AccessKey string `json:"ak"` - TokenID string `json:"token_id"` -} - -func (m *Manager) exchangeAndProvision( - ctx context.Context, - payload loginGrantPayload, - identity *Credential, - options LoginOptions, -) (*Credential, error) { - if payload.ExpireAt > 0 && payload.ExpireAt <= m.now().Unix() { - return nil, errors.New("网页授权已过期，请重新登录") - } - jar, err := cookiejar.New(nil) - if err != nil { - return nil, errors.New("初始化临时网页登录会话失败") - } - client := *m.httpClient - client.Jar = jar - client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { - return http.ErrUseLastResponse - } - - exchangeRequest := struct { - Grant string `json:"grant"` - Secret string `json:"random_secret_key"` - }{Grant: payload.Grant, Secret: payload.RandomSecretKey} - exchange, err := doJSON[exchangeData](ctx, &client, m.authBaseURL, exchangeGrantPath, exchangeRequest, "交换网页授权") - if err != nil { - return nil, err - } - if strings.TrimSpace(exchange.UID) == "" || !constantTimeEqual(exchange.Scope, loginGrantScope) { - return nil, errors.New("网页授权响应缺少有效的用户身份") - } - exchange.UID = strings.TrimSpace(exchange.UID) - actualScope := credentialScope(exchange.UID, identity.DeviceID) - if expected := strings.TrimSpace(options.ExpectedCredentialScope); expected != "" && - !constantTimeEqual(expected, actualScope) { - return nil, ErrCredentialAccountMismatch - } - if len(jar.Cookies(m.authBaseURL)) == 0 { - return nil, errors.New("网页授权响应没有建立临时登录会话") - } - - query, err := doJSON[queryAccessKeyData](ctx, &client, m.authBaseURL, queryAccessKeyPath, nil, "查询 CLI 凭证") - if err != nil { - return nil, err - } - var rotatedTokenIDs []string - if options.ForceRefresh { - // A stored TokenID only identifies this device's token inside the - // account that originally issued it. Never use it as a destructive - // selector after the browser has switched to another UID. - if identity.UID != "" && constantTimeEqual(identity.UID, exchange.UID) { - rotatedTokenIDs = managedTokenIDs(query.AccessTokens, identity) - } - if len(rotatedTokenIDs) > 0 { - deleteRequest := struct { - AKIDs []string `json:"ak_ids"` - }{AKIDs: rotatedTokenIDs} - if _, err := doJSON[struct{}](ctx, &client, m.authBaseURL, deleteAccessKeyPath, deleteRequest, "轮换旧的 CLI 凭证"); err != nil { - return nil, err - } - } - } else { - selected, err := m.selectManagedToken(query.AccessTokens, identity) - if err != nil { - return nil, err - } - if selected != nil { - return credentialFromToken(identity, exchange.UID, selected), nil - } - } - - expiredAt := m.now().Add(DefaultCredentialLifetime).Unix() - generateRequest := struct { - TokenName string `json:"token_name"` - TokenDesc string `json:"token_desc"` - ExpiredAt int64 `json:"expired_at"` - }{ - TokenName: identity.TokenName, - TokenDesc: "Pippit Tool CLI browser login", - ExpiredAt: expiredAt, - } - generated, err := doJSON[generateAccessKeyData](ctx, &client, m.authBaseURL, generateAccessKeyPath, generateRequest, "创建 CLI 凭证") - if err != nil { - var responseErr *apiResponseError - if errors.As(err, &responseErr) && responseErr.ret == "3" { - return nil, errors.New("当前账号暂不具备创建 CLI Access Key 的权限，请升级、联系管理员或使用已有 XYQ_ACCESS_KEY") - } - if errors.As(err, &responseErr) && responseErr.ret != "" { - return nil, errors.New("无法创建新的 CLI Access Key；请在个人设置中检查 Access Key 数量上限和账号权限后重试") - } - return nil, err - } - if strings.TrimSpace(generated.AccessKey) == "" || strings.TrimSpace(generated.TokenID) == "" { - return nil, errors.New("创建 CLI 凭证后服务端未返回完整结果") - } - credential := cloneCredential(identity) - credential.AccessKey = strings.TrimSpace(generated.AccessKey) - credential.TokenID = strings.TrimSpace(generated.TokenID) - credential.UID = exchange.UID - credential.CredentialScope = actualScope - credential.ExpiredAt = expiredAt - if options.ForceRefresh && identity.UID == credential.UID && identity.AccessKey != "" && - constantTimeEqual(identity.AccessKey, credential.AccessKey) { - return nil, errors.New("服务端未轮换已失效的 CLI Access Key，已拒绝继续使用旧凭证") - } - for _, tokenID := range rotatedTokenIDs { - if constantTimeEqual(tokenID, credential.TokenID) { - return nil, errors.New("服务端未轮换已失效的 CLI 凭证编号，已拒绝继续使用旧凭证") - } - } - if err := validateCredential(credential); err != nil { - return nil, errors.New("创建的 CLI 凭证格式无效") - } - return credential, nil -} - -func managedTokenIDs(tokens []accessToken, identity *Credential) []string { - result := make([]string, 0, 1) - if identity == nil || strings.TrimSpace(identity.TokenID) == "" { - return result - } - for _, token := range tokens { - id := strings.TrimSpace(token.ID) - // QueryAk is scoped by the exchanged browser UID, while TokenID comes - // from this device's securely stored credential. Their exact match is - // the destructive-operation boundary even if the user renamed the token. - if id == "" || !constantTimeEqual(id, identity.TokenID) { - continue - } - result = append(result, id) - break - } - return result -} - -func (m *Manager) selectManagedToken(tokens []accessToken, identity *Credential) (*accessToken, error) { - valid := make([]accessToken, 0, 1) - minimumExpiry := m.now().Add(m.ensureTTL()).Unix() - if identity.TokenID != "" { - for index := range tokens { - if constantTimeEqual(tokens[index].ID, identity.TokenID) && usableAccessToken(tokens[index], minimumExpiry) { - // TokenID is the stable device-owned identity. Prefer it before - // matching the display name because users may rename a token in UI. - return &tokens[index], nil - } - } - } - for _, token := range tokens { - if !constantTimeEqual(token.Name, identity.TokenName) || !usableAccessToken(token, minimumExpiry) { - continue - } - valid = append(valid, token) - } - if len(valid) == 0 { - return nil, nil - } - if len(valid) != 1 { - return nil, errors.New("检测到多个同设备 CLI 凭证，拒绝自动选择；请在个人设置中清理重复项后重试") - } - return &valid[0], nil -} - -func usableAccessToken(token accessToken, minimumExpiry int64) bool { - return token.Status == "enable" && strings.TrimSpace(token.Token) != "" && token.ExpiredAt > minimumExpiry -} - -func credentialFromToken(identity *Credential, uid string, token *accessToken) *Credential { - credential := cloneCredential(identity) - credential.AccessKey = strings.TrimSpace(token.Token) - credential.TokenID = strings.TrimSpace(token.ID) - credential.UID = strings.TrimSpace(uid) - credential.CredentialScope = credentialScope(credential.UID, credential.DeviceID) - credential.ExpiredAt = token.ExpiredAt - return credential -} - -func doJSON[T any](ctx context.Context, client *http.Client, baseURL *url.URL, path string, body any, operation string) (T, error) { - var zero T - requestURL := *baseURL - requestURL.Path = path - requestURL.RawPath = "" - requestURL.RawQuery = "" - requestURL.Fragment = "" - - var reader io.Reader - if body != nil { - payload, err := json.Marshal(body) - if err != nil { - return zero, redactedOperationError(operation) - } - reader = bytes.NewReader(payload) - } - request, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL.String(), reader) - if err != nil { - return zero, redactedOperationError(operation) - } - request.Header.Set("Accept", "application/json") - request.Header.Set("User-Agent", "Pippit-CLI/1.0") - request.Header.Set("appvr", "1.1.4") - request.Header.Set("entrance-from", "web") - request.Header.Set("appid", "795647") - if body != nil { - request.Header.Set("Content-Type", "application/json") - } - response, err := client.Do(request) - if err != nil { - return zero, redactedOperationError(operation) - } - defer response.Body.Close() - responseBody, err := io.ReadAll(io.LimitReader(response.Body, maxAPIResponseBytes+1)) - if err != nil || len(responseBody) > maxAPIResponseBytes { - return zero, redactedOperationError(operation) - } - if response.StatusCode < 200 || response.StatusCode >= 300 { - return zero, &apiResponseError{operation: operation, httpStatus: response.StatusCode} - } - envelope := apiEnvelope[T]{} - if err := json.Unmarshal(responseBody, &envelope); err != nil { - return zero, fmt.Errorf("%s响应格式无效", operation) - } - if !successfulRet(envelope.Ret) { - return zero, &apiResponseError{operation: operation, ret: safeRet(envelope.Ret)} - } - return envelope.Data, nil -} - -func successfulRet(value json.RawMessage) bool { - trimmed := strings.TrimSpace(string(value)) - return trimmed == "" || trimmed == "null" || trimmed == `""` || trimmed == "0" || trimmed == `"0"` -} - -func safeRet(value json.RawMessage) string { - trimmed := strings.Trim(strings.TrimSpace(string(value)), `"`) - if _, err := strconv.ParseInt(trimmed, 10, 64); err == nil && len(trimmed) <= 20 { - return trimmed - } - return "unknown" -} diff --git a/internal/auth/identity.go b/internal/auth/identity.go index 4882800..2f17a76 100644 --- a/internal/auth/identity.go +++ b/internal/auth/identity.go @@ -6,7 +6,6 @@ import ( "crypto/subtle" "encoding/base64" "errors" - "fmt" "io" "strings" ) @@ -36,14 +35,23 @@ func validDeviceID(deviceID string) bool { constantTimeEqual(deviceID, base64.RawURLEncoding.EncodeToString(decoded)) } -func tokenNameForDevice(deviceID string) string { - digest := sha256.Sum256([]byte(deviceID)) - return "pippit-tool-cli-" + base64.RawURLEncoding.EncodeToString(digest[:16]) +func credentialScope(uid, deviceID string) string { + return "pippit-tool-cli:user:" + accountBinding(uid) + ":device:" + deviceID } -func credentialScope(uid, deviceID string) string { +func accountBinding(uid string) string { digest := sha256.Sum256([]byte(strings.TrimSpace(uid))) - return "pippit-tool-cli:user:" + base64.RawURLEncoding.EncodeToString(digest[:16]) + ":device:" + deviceID + return base64.RawURLEncoding.EncodeToString(digest[:16]) +} + +func accountBindingFromCredentialScope(scope, deviceID string) (string, bool) { + prefix := "pippit-tool-cli:user:" + suffix := ":device:" + deviceID + if !strings.HasPrefix(scope, prefix) || !strings.HasSuffix(scope, suffix) { + return "", false + } + binding := strings.TrimSuffix(strings.TrimPrefix(scope, prefix), suffix) + return binding, validAccountBinding(binding) } func legacyCredentialScope(deviceID string) string { @@ -62,9 +70,8 @@ func newIdentity(reader io.Reader) (*Credential, error) { return nil, errors.New("生成本机登录设备标识失败") } return &Credential{ - Version: credentialVersion, - DeviceID: deviceID, - TokenName: tokenNameForDevice(deviceID), + Version: credentialVersion, + DeviceID: deviceID, }, nil } @@ -73,20 +80,10 @@ func identityOnly(credential *Credential) *Credential { return nil } return &Credential{ - Version: credential.Version, - DeviceID: credential.DeviceID, - TokenName: credential.TokenName, - // TokenID is not an authentication secret. Keeping this exact remote - // reference lets a later login reuse or rotate the same device token - // without consuming another per-account AK slot. + Version: credential.Version, + DeviceID: credential.DeviceID, + // TokenID is a non-secret exact selector used by the Web page to reuse the + // same remote token after logout instead of consuming another AK slot. TokenID: credential.TokenID, } } - -func redactedOperationError(operation string) error { - operation = strings.TrimSpace(operation) - if operation == "" { - operation = "授权操作" - } - return fmt.Errorf("%s失败，请稍后重试", operation) -} diff --git a/internal/auth/loopback.go b/internal/auth/loopback.go index ac34136..ca64c27 100644 --- a/internal/auth/loopback.go +++ b/internal/auth/loopback.go @@ -2,6 +2,7 @@ package auth import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -17,30 +18,48 @@ import ( const maxCallbackBodyBytes = 64 << 10 -type loginGrantPayload struct { +type accessKeyPayload struct { Type string `json:"type"` - Grant string `json:"grant"` + AccessKey string `json:"access_key"` + UID string `json:"uid"` + TokenID string `json:"token_id"` + ExpiredAt int64 `json:"expired_at"` RandomSecretKey string `json:"random_secret_key"` - ExpireAt int64 `json:"expire_at,omitempty"` Source string `json:"source"` CallbackURL string `json:"callback_url"` } type browserFlow struct { - loginURL string - callbackURL string - secret string - state string - source string - origin string - listener net.Listener - server *http.Server - payload chan loginGrantPayload - serveErr chan error - closeOnce sync.Once + loginURL string + callbackURL string + secret string + state string + source string + origin string + listener net.Listener + server *http.Server + payload chan accessKeyPayload + serveErr chan error + callbackOnce sync.Once + closeOnce sync.Once } -func startBrowserFlow(authBaseURL *url.URL, randomReader io.Reader) (*browserFlow, error) { +func startBrowserFlow(authBaseURL *url.URL, randomReader io.Reader, deviceID, tokenID, expectedAccount string, forceRefresh bool) (*browserFlow, error) { + if !validDeviceID(deviceID) { + return nil, errors.New("本机登录设备标识无效") + } + if tokenID != "" && !validTokenID(tokenID) { + return nil, errors.New("本机 CLI 凭证编号无效") + } + if forceRefresh && tokenID == "" { + return nil, errors.New("无法确认需要轮换的本机 CLI 凭证编号") + } + if expectedAccount != "" && !validAccountBinding(expectedAccount) { + return nil, errors.New("本机登录账号绑定无效") + } + if forceRefresh && expectedAccount == "" { + return nil, errors.New("无法确认需要轮换的本机 CLI 登录账号") + } secret, err := randomEncoded(randomReader, randomBindingBytes) if err != nil { return nil, errors.New("生成网页授权绑定信息失败") @@ -64,7 +83,7 @@ func startBrowserFlow(authBaseURL *url.URL, randomReader io.Reader) (*browserFlo callback.RawQuery = callbackQuery.Encode() loginURL := *authBaseURL - loginURL.Path = loginExportPath + loginURL.Path = loginPagePath loginURL.RawPath = "" loginURL.RawQuery = "" loginURL.Fragment = "" @@ -72,6 +91,16 @@ func startBrowserFlow(authBaseURL *url.URL, randomReader io.Reader) (*browserFlo query.Set("callback", callback.String()) query.Set("random_secret_key", secret) query.Set("source", loginSource) + query.Set("device_id", deviceID) + if tokenID != "" { + query.Set("token_id", tokenID) + } + if expectedAccount != "" { + query.Set("expected_account", expectedAccount) + } + if forceRefresh { + query.Set("force", "1") + } loginURL.RawQuery = query.Encode() flow := &browserFlow{ @@ -82,7 +111,7 @@ func startBrowserFlow(authBaseURL *url.URL, randomReader io.Reader) (*browserFlo source: loginSource, origin: originOf(authBaseURL), listener: listener, - payload: make(chan loginGrantPayload, 1), + payload: make(chan accessKeyPayload, 1), serveErr: make(chan error, 1), } mux := http.NewServeMux() @@ -104,20 +133,20 @@ func startBrowserFlow(authBaseURL *url.URL, randomReader io.Reader) (*browserFlo return flow, nil } -func (f *browserFlow) wait(ctx context.Context) (loginGrantPayload, error) { +func (f *browserFlow) wait(ctx context.Context) (accessKeyPayload, error) { select { case payload := <-f.payload: return payload, nil case err, open := <-f.serveErr: if open && err != nil { - return loginGrantPayload{}, errors.New("本机网页授权回调异常退出") + return accessKeyPayload{}, errors.New("本机网页授权回调异常退出") } - return loginGrantPayload{}, errors.New("本机网页授权回调已关闭") + return accessKeyPayload{}, errors.New("本机网页授权回调已关闭") case <-ctx.Done(): if errors.Is(ctx.Err(), context.DeadlineExceeded) { - return loginGrantPayload{}, errors.New("等待网页授权超时，请重新登录") + return accessKeyPayload{}, errors.New("等待网页授权超时，请重新登录") } - return loginGrantPayload{}, ctx.Err() + return accessKeyPayload{}, ctx.Err() } } @@ -164,7 +193,7 @@ func (f *browserFlow) handleCallback(writer http.ResponseWriter, request *http.R reader := http.MaxBytesReader(writer, request.Body, maxCallbackBodyBytes) decoder := json.NewDecoder(reader) decoder.DisallowUnknownFields() - payload := loginGrantPayload{} + payload := accessKeyPayload{} if err := decoder.Decode(&payload); err != nil { http.Error(writer, "invalid callback payload", http.StatusBadRequest) return @@ -173,21 +202,51 @@ func (f *browserFlow) handleCallback(writer http.ResponseWriter, request *http.R http.Error(writer, "invalid callback payload", http.StatusBadRequest) return } - if payload.Type != "login_grant" || strings.TrimSpace(payload.Grant) == "" || + if payload.Type != "access_key" || !validCallbackValue(payload.AccessKey, 4096) || + !validCallbackValue(payload.UID, 256) || !validTokenID(payload.TokenID) || payload.ExpiredAt <= 0 || !constantTimeEqual(payload.RandomSecretKey, f.secret) || !constantTimeEqual(payload.Source, f.source) || !constantTimeEqual(payload.CallbackURL, f.callbackURL) { http.Error(writer, "callback binding mismatch", http.StatusBadRequest) return } - select { - case f.payload <- payload: + accepted := false + f.callbackOnce.Do(func() { + f.payload <- payload + accepted = true + }) + if accepted { writer.Header().Set("Content-Type", "application/json") writer.WriteHeader(http.StatusOK) _, _ = writer.Write([]byte(`{"ok":true}`)) - default: - http.Error(writer, "callback already received", http.StatusConflict) + return } + http.Error(writer, "callback already received", http.StatusConflict) +} + +func validCallbackValue(value string, maxLength int) bool { + return value != "" && len(value) <= maxLength && strings.TrimSpace(value) == value +} + +func validTokenID(value string) bool { + if len(value) == 0 || len(value) > 128 { + return false + } + for index := range len(value) { + character := value[index] + if (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') || + (character >= '0' && character <= '9') || strings.ContainsRune(":._-", rune(character)) { + continue + } + return false + } + return true +} + +func validAccountBinding(value string) bool { + decoded, err := base64.RawURLEncoding.DecodeString(value) + return err == nil && len(decoded) == 16 && len(value) == 22 && + constantTimeEqual(value, base64.RawURLEncoding.EncodeToString(decoded)) } func (f *browserFlow) validRequestTarget(request *http.Request) bool { diff --git a/internal/auth/manager.go b/internal/auth/manager.go index 8a9271d..5bf0e8f 100644 --- a/internal/auth/manager.go +++ b/internal/auth/manager.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "net/http" "net/url" "strings" "sync" @@ -18,7 +17,6 @@ import ( type Manager struct { cfg *config.Config store CredentialStore - httpClient *http.Client authBaseURL *url.URL random io.Reader now func() time.Time @@ -37,23 +35,6 @@ func WithCredentialStore(store CredentialStore) ManagerOption { } } -func WithHTTPClient(client *http.Client) ManagerOption { - return func(manager *Manager) { - if client != nil { - manager.httpClient = client - } - } -} - -func withAuthBaseURLForTest(rawURL string) ManagerOption { - return func(manager *Manager) { - parsed, err := url.Parse(rawURL) - if err == nil { - manager.authBaseURL = parsed - } - } -} - func withRandomReaderForTest(reader io.Reader) ManagerOption { return func(manager *Manager) { if reader != nil { @@ -70,20 +51,15 @@ func withClockForTest(now func() time.Time) ManagerOption { } } -// NewManager always uses the canonical production auth origin. cfg.BaseURL and -// cfg.PPEEnv intentionally do not affect browser grants, login cookies, or AK -// provisioning; PPE routing applies only after a dedicated AK has been issued. +// NewManager always uses the canonical production login page. cfg.BaseURL and +// cfg.PPEEnv intentionally do not affect browser login; PPE routing applies +// only to business requests after the page has returned an Access Key. func NewManager(cfg *config.Config, options ...ManagerOption) *Manager { serviceName := config.DefaultAuthStoreServiceName authBaseURL, _ := url.Parse(config.DefaultBaseURL) - timeout := config.DefaultHTTPTimeout - if cfg != nil && cfg.HTTPTimeout > 0 { - timeout = cfg.HTTPTimeout - } manager := &Manager{ cfg: cfg, store: NewDefaultCredentialStore(serviceName), - httpClient: &http.Client{Timeout: timeout}, authBaseURL: authBaseURL, random: rand.Reader, now: time.Now, @@ -123,7 +99,21 @@ func (m *Manager) Login(ctx context.Context, options LoginOptions) (*Credential, if err != nil { return nil, err } - flow, err := startBrowserFlow(m.authBaseURL, m.random) + forceRefresh := options.ForceRefresh && strings.TrimSpace(identity.TokenID) != "" + expectedAccount := "" + if identity.UID != "" { + expectedAccount = accountBinding(identity.UID) + } + if expectedScope := strings.TrimSpace(options.ExpectedCredentialScope); expectedScope != "" { + scopeAccount, ok := accountBindingFromCredentialScope(expectedScope, identity.DeviceID) + if !ok || (expectedAccount != "" && !constantTimeEqual(expectedAccount, scopeAccount)) { + return nil, ErrCredentialAccountMismatch + } + expectedAccount = scopeAccount + } + flow, err := startBrowserFlow( + m.authBaseURL, m.random, identity.DeviceID, identity.TokenID, expectedAccount, forceRefresh, + ) if err != nil { return nil, err } @@ -149,10 +139,22 @@ func (m *Manager) Login(ctx context.Context, options LoginOptions) (*Credential, if err != nil { return nil, err } - writeProgress(options.Progress, "网页授权已完成，正在准备本机专属 CLI 凭证…") - credential, err := m.exchangeAndProvision(waitCtx, payload, identity, options) - if err != nil { - return nil, err + writeProgress(options.Progress, "网页授权已完成，正在安全保存本机 CLI 凭证…") + credential := credentialFromCallback(identity, payload) + if credential.ExpiredAt <= m.now().Add(m.ensureTTL()).Unix() { + return nil, ErrCredentialExpired + } + if expected := strings.TrimSpace(options.ExpectedCredentialScope); expected != "" && + !constantTimeEqual(expected, credential.CredentialScope) { + return nil, ErrCredentialAccountMismatch + } + if forceRefresh && + ((identity.AccessKey != "" && constantTimeEqual(identity.AccessKey, credential.AccessKey)) || + constantTimeEqual(identity.TokenID, credential.TokenID)) { + return nil, errors.New("网页授权未轮换已失效的 CLI Access Key，已拒绝继续使用旧凭证") + } + if err := validateCredential(credential); err != nil { + return nil, errors.New("网页返回的 CLI 凭证格式无效") } if err := m.saveCredential(waitCtx, credential); err != nil { return nil, err @@ -161,6 +163,16 @@ func (m *Manager) Login(ctx context.Context, options LoginOptions) (*Credential, return cloneCredential(credential), nil } +func credentialFromCallback(identity *Credential, payload accessKeyPayload) *Credential { + credential := cloneCredential(identity) + credential.AccessKey = payload.AccessKey + credential.TokenID = payload.TokenID + credential.UID = payload.UID + credential.ExpiredAt = payload.ExpiredAt + credential.CredentialScope = credentialScope(payload.UID, identity.DeviceID) + return credential +} + func (m *Manager) Status(ctx context.Context) (*Status, error) { if m != nil && m.cfg != nil && strings.TrimSpace(m.cfg.AccessKey) != "" { return &Status{LoggedIn: true, Source: "environment"}, nil @@ -305,7 +317,7 @@ func normalizeCredential(credential *Credential) *Credential { } func (m *Manager) validate() error { - if m == nil || m.store == nil || m.httpClient == nil || m.authBaseURL == nil || m.random == nil || m.now == nil { + if m == nil || m.store == nil || m.authBaseURL == nil || m.random == nil || m.now == nil { return errors.New("小云雀 CLI 授权管理器未正确初始化") } if m.authBaseURL.Scheme != "https" && !(m.authBaseURL.Scheme == "http" && isLoopbackHost(m.authBaseURL.Hostname())) { diff --git a/internal/auth/store.go b/internal/auth/store.go index 81d4d47..d892745 100644 --- a/internal/auth/store.go +++ b/internal/auth/store.go @@ -99,7 +99,9 @@ type storedCredential struct { Version int `json:"version"` DeviceID string `json:"device_id"` CredentialScope string `json:"credential_scope"` - TokenName string `json:"token_name"` + // LegacyTokenName is decoded only for compatibility with credentials written + // by the first browser-auth beta. New records intentionally omit it. + LegacyTokenName string `json:"token_name,omitempty"` AccessKey string `json:"access_key,omitempty"` TokenID string `json:"token_id,omitempty"` UID string `json:"uid,omitempty"` @@ -212,7 +214,6 @@ func encodeCredential(credential *Credential) ([]byte, error) { Version: credential.Version, DeviceID: credential.DeviceID, CredentialScope: credential.CredentialScope, - TokenName: credential.TokenName, AccessKey: credential.AccessKey, TokenID: credential.TokenID, UID: credential.UID, @@ -236,7 +237,6 @@ func decodeCredential(payload []byte) (*Credential, error) { Version: record.Version, DeviceID: record.DeviceID, CredentialScope: record.CredentialScope, - TokenName: record.TokenName, AccessKey: record.AccessKey, TokenID: record.TokenID, UID: record.UID, @@ -280,18 +280,18 @@ func validateCredential(credential *Credential) error { if !validDeviceID(credential.DeviceID) { return errors.New("本机登录设备标识无效") } - if !constantTimeEqual(credential.TokenName, tokenNameForDevice(credential.DeviceID)) { - return errors.New("本机登录凭证作用域无效") - } if credential.AccessKey == "" { - if strings.TrimSpace(credential.TokenID) != credential.TokenID || credential.UID != "" || credential.ExpiredAt != 0 || - (credential.CredentialScope != "" && !constantTimeEqual(credential.CredentialScope, legacyCredentialScope(credential.DeviceID))) { + // TokenID is non-secret. Keeping it lets an explicit force login select + // exactly this device token without guessing by display name. + if (credential.TokenID != "" && !validTokenID(credential.TokenID)) || credential.UID != "" || + credential.ExpiredAt != 0 || credential.CredentialScope != "" { return errors.New("本机登录凭证不完整") } return nil } - if strings.TrimSpace(credential.AccessKey) != credential.AccessKey || - credential.TokenID == "" || credential.UID == "" || credential.ExpiredAt <= 0 { + if strings.TrimSpace(credential.AccessKey) != credential.AccessKey || len(credential.AccessKey) > 4096 || + !validTokenID(credential.TokenID) || + credential.UID == "" || len(credential.UID) > 256 || strings.TrimSpace(credential.UID) != credential.UID || credential.ExpiredAt <= 0 { return errors.New("本机登录凭证不完整") } expectedScope := credentialScope(credential.UID, credential.DeviceID) diff --git a/internal/auth/types.go b/internal/auth/types.go index d84518d..35d390a 100644 --- a/internal/auth/types.go +++ b/internal/auth/types.go @@ -8,19 +8,14 @@ import ( ) const ( - loginExportPath = "/cli/login-export" - callbackPath = "/xyq/callback/save_session" - exchangeGrantPath = "/api/web/v1/auth/exchange_cli_login_grant" - queryAccessKeyPath = "/api/biz/v1/user/query_ak" - generateAccessKeyPath = "/api/biz/v1/user/generate_ak" - deleteAccessKeyPath = "/api/biz/v1/user/delete_ak" - loginSource = "pippit-tool-cli" - credentialVersion = 1 - deviceIDBytes = 32 - randomBindingBytes = 32 + loginPagePath = "/cli/pippit-tool-login" + callbackPath = "/pippit-tool/callback" + loginSource = "pippit-tool-cli" + credentialVersion = 1 + deviceIDBytes = 32 + randomBindingBytes = 32 - DefaultLoginTimeout = 5 * time.Minute - DefaultCredentialLifetime = 365 * 24 * time.Hour + DefaultLoginTimeout = 5 * time.Minute ) var ( @@ -38,7 +33,6 @@ type Credential struct { Version int `json:"version"` DeviceID string `json:"device_id"` CredentialScope string `json:"credential_scope"` - TokenName string `json:"token_name"` AccessKey string `json:"-"` TokenID string `json:"token_id,omitempty"` UID string `json:"uid,omitempty"` @@ -58,13 +52,12 @@ type LoginOptions struct { OpenURL func(string) error Progress io.Writer Timeout time.Duration - // ForceRefresh rotates every remote token owned by this device identity - // before provisioning a replacement. It is used after an explicit 401 so - // a successful browser login can never return the just-rejected AK again. + // ForceRefresh asks the browser page to rotate this device's rejected AK. + // The CLI never calls QueryAk, DeleteAk, or GenerateAk itself. ForceRefresh bool // ExpectedCredentialScope binds reauthentication to the UID and device that - // started a durable operation. A different browser account fails before any - // Access Key is deleted or generated. + // started a durable operation. A different browser account fails before the + // returned Access Key is saved. ExpectedCredentialScope string } From 8eab4261795da2ff3095f6aa3024a1131b608d15 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 12:35:38 +0800 Subject: [PATCH 47/48] test(auth): cover direct browser credential delivery Co-authored-by: Codex <codex@openai.com> --- internal/auth/auth_test.go | 994 +++++++++++++++---------------------- 1 file changed, 413 insertions(+), 581 deletions(-) diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 06ee9ee..85e17d5 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -9,10 +9,10 @@ import ( "fmt" "io" "net/http" - "net/http/httptest" "net/url" "os" "path/filepath" + "runtime" "strings" "sync" "testing" @@ -79,9 +79,13 @@ func (s *memoryCredentialStore) Delete(ctx context.Context) error { return nil } -func TestBrowserFlowRequiresExactBoundCallbackAndCORS(t *testing.T) { - authURL, _ := url.Parse("https://xyq.jianying.com") - flow, err := startBrowserFlow(authURL, bytes.NewReader(bytes.Repeat([]byte{0x42}, 64))) +func TestBrowserFlowUsesDedicatedPageAndStrictBoundCallback(t *testing.T) { + authURL, _ := url.Parse(config.DefaultBaseURL) + deviceID, _ := randomEncoded(bytes.NewReader(bytes.Repeat([]byte{0x21}, deviceIDBytes)), deviceIDBytes) + flow, err := startBrowserFlow( + authURL, bytes.NewReader(bytes.Repeat([]byte{0x42}, 2*randomBindingBytes)), + deviceID, "saved-token-id", accountBinding("saved-user"), true, + ) if err != nil { t.Fatal(err) } @@ -91,562 +95,418 @@ func TestBrowserFlowRequiresExactBoundCallbackAndCORS(t *testing.T) { if err != nil { t.Fatal(err) } - if got := loginURL.Query().Get("source"); got != loginSource { - t.Fatalf("source = %q, want %q", got, loginSource) + if loginURL.Scheme+"://"+loginURL.Host != config.DefaultBaseURL || loginURL.Path != loginPagePath { + t.Fatalf("login URL target = %s://%s%s", loginURL.Scheme, loginURL.Host, loginURL.Path) } - if got := loginURL.Query().Get("ppe_env"); got != "" { - t.Fatalf("ppe_env = %q, want absent", got) + query := loginURL.Query() + if query.Get("source") != loginSource || query.Get("device_id") != deviceID || query.Get("force") != "1" || + query.Get("token_id") != "saved-token-id" || query.Get("expected_account") != accountBinding("saved-user") { + t.Fatalf("login URL public binding is incomplete: %v", query) } - for _, name := range []string{"random_secret_key"} { - decoded, err := base64.RawURLEncoding.DecodeString(loginURL.Query().Get(name)) - if err != nil || len(decoded) < randomBindingBytes { - t.Fatalf("%s is not at least %d random bytes", name, randomBindingBytes) + for _, field := range []string{"random_secret_key"} { + decoded, decodeErr := base64.RawURLEncoding.DecodeString(query.Get(field)) + if decodeErr != nil || len(decoded) != randomBindingBytes || len(query.Get(field)) != 43 { + t.Fatalf("%s is not canonical 32-byte Base64URL", field) } } - callback, err := url.Parse(flow.callbackURL) + callbackURL := query.Get("callback") + parsedCallback, err := url.Parse(callbackURL) if err != nil { t.Fatal(err) } - decodedState, err := base64.RawURLEncoding.DecodeString(callback.Query().Get("state")) - if err != nil || len(decodedState) < randomBindingBytes { - t.Fatalf("state is not at least %d random bytes", randomBindingBytes) + state := parsedCallback.Query().Get("state") + decodedState, decodeErr := base64.RawURLEncoding.DecodeString(state) + if parsedCallback.Scheme != "http" || parsedCallback.Hostname() != "127.0.0.1" || parsedCallback.Path != callbackPath || + decodeErr != nil || len(decodedState) != randomBindingBytes || len(state) != 43 { + t.Fatalf("callback is not canonical: %q", callbackURL) } - preflight, _ := http.NewRequest(http.MethodOptions, flow.callbackURL, nil) - preflight.Header.Set("Origin", "https://xyq.jianying.com") - preflight.Header.Set("Access-Control-Request-Method", http.MethodPost) - preflight.Header.Set("Access-Control-Request-Headers", "content-type") - preflightResponse, err := http.DefaultClient.Do(preflight) - if err != nil { - t.Fatal(err) - } - preflightResponse.Body.Close() - if preflightResponse.StatusCode != http.StatusNoContent { - t.Fatalf("preflight status = %d", preflightResponse.StatusCode) + payload := accessKeyPayload{ + Type: "access_key", + AccessKey: "callback-ak-secret", + UID: "12345", + TokenID: "token-id", + ExpiredAt: time.Now().Add(time.Hour).Unix(), + RandomSecretKey: query.Get("random_secret_key"), + Source: loginSource, + CallbackURL: callbackURL, } - if got := preflightResponse.Header.Get("Access-Control-Allow-Origin"); got != "https://xyq.jianying.com" { - t.Fatalf("allow origin = %q", got) - } - if got := preflightResponse.Header.Get("Access-Control-Allow-Private-Network"); got != "true" { - t.Fatalf("allow private network = %q", got) + if status := sendCallback(t, callbackURL, "https://evil.example", payload); status != http.StatusForbidden { + t.Fatalf("wrong origin status = %d", status) } - - payload := loginGrantPayload{ - Type: "login_grant", - Grant: "one-time-grant", - RandomSecretKey: flow.secret, - Source: flow.source, - CallbackURL: flow.callbackURL, + wrongSecret := payload + wrongSecret.RandomSecretKey = "wrong-secret" + if status := sendCallback(t, callbackURL, config.DefaultBaseURL, wrongSecret); status != http.StatusBadRequest { + t.Fatalf("wrong secret status = %d", status) } - wrong := payload - wrong.Source = "other-cli" - if status := postCallback(t, flow.callbackURL, flow.origin, wrong); status != http.StatusBadRequest { - t.Fatalf("wrong binding status = %d", status) + if status := postRawCallback(t, callbackURL, config.DefaultBaseURL, `{"type":"access_key","unknown":true}`); status != http.StatusBadRequest { + t.Fatalf("unknown field status = %d", status) } - wrongStateURL := *callback - wrongStateQuery := wrongStateURL.Query() - wrongStateQuery.Set("state", "wrong-state") - wrongStateURL.RawQuery = wrongStateQuery.Encode() - if status := postCallback(t, wrongStateURL.String(), flow.origin, payload); status != http.StatusBadRequest { - t.Fatalf("wrong state status = %d", status) + if status := sendCallback(t, callbackURL+"&state=duplicate", config.DefaultBaseURL, payload); status != http.StatusBadRequest { + t.Fatalf("duplicate state status = %d", status) } - if status := postCallback(t, flow.callbackURL, "https://attacker.invalid", payload); status != http.StatusForbidden { - t.Fatalf("wrong origin status = %d", status) + if status := sendPreflight(t, callbackURL, config.DefaultBaseURL); status != http.StatusNoContent { + t.Fatalf("preflight status = %d", status) } - if status := postCallback(t, flow.callbackURL, flow.origin, payload); status != http.StatusOK { + if status := sendCallback(t, callbackURL, config.DefaultBaseURL, payload); status != http.StatusOK { t.Fatalf("valid callback status = %d", status) } got, err := flow.wait(context.Background()) - if err != nil { - t.Fatal(err) + if err != nil || got.AccessKey != payload.AccessKey || got.UID != payload.UID || got.TokenID != payload.TokenID { + t.Fatalf("callback payload = %#v, %v", credentialPayloadWithoutSecret(got), err) } - if got.Grant != payload.Grant { - t.Fatalf("grant = %q", got.Grant) + if status := sendCallback(t, callbackURL, config.DefaultBaseURL, payload); status != http.StatusConflict { + t.Fatalf("replayed callback status = %d", status) } } -func TestManagerLoginReusesOnlyExactDeviceToken(t *testing.T) { +func TestManagerLoginStoresPageIssuedCredentialWithoutServerExchange(t *testing.T) { fixedNow := time.Unix(1_800_000_000, 0) store := &memoryCredentialStore{} - var generated bool - var expectedName string - server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - assertAuthHeaders(t, request) - if request.Header.Get("x-use-ppe") != "" || request.Header.Get("x-tt-env") != "" { - t.Errorf("auth request unexpectedly carried PPE headers") - } - switch request.URL.Path { - case exchangeGrantPath: - var body map[string]string - _ = json.NewDecoder(request.Body).Decode(&body) - if body["grant"] != "grant-value" || body["random_secret_key"] == "" { - t.Errorf("unexpected exchange body") - } - http.SetCookie(writer, &http.Cookie{Name: "session", Value: "cookie-secret", Path: "/", Secure: true, HttpOnly: true}) - writeEnvelope(writer, map[string]any{"uid": "123", "scope": loginGrantScope}) - case queryAccessKeyPath: - cookie, err := request.Cookie("session") - if err != nil || cookie.Value != "cookie-secret" { - t.Errorf("query did not receive temporary exchange cookie") - } - credential, err := store.Load(context.Background()) - if err != nil { - t.Errorf("load identity: %v", err) - return - } - expectedName = credential.TokenName - writeEnvelope(writer, map[string]any{"access_token_list": []map[string]any{ - {"ak_id": "foreign-id", "token": "foreign-ak", "expired_at": fixedNow.Add(time.Hour).Unix(), "token_name": "someone-else", "token_status": "enable"}, - {"ak_id": "managed-id", "token": "managed-ak", "expired_at": fixedNow.Add(time.Hour).Unix(), "token_name": credential.TokenName, "token_status": "enable"}, - }}) - case generateAccessKeyPath: - generated = true - http.Error(writer, "must not generate", http.StatusInternalServerError) - default: - http.NotFound(writer, request) - } - })) - defer server.Close() - cfg := config.Load() cfg.AccessKey = "" - cfg.PPEEnv = "ppe_must_not_reach_auth" - manager := NewManager(cfg, + manager := NewManager( + cfg, WithCredentialStore(store), - WithHTTPClient(server.Client()), - withAuthBaseURLForTest(server.URL), + withRandomReaderForTest(bytes.NewReader(bytes.Repeat([]byte{0x31}, deviceIDBytes+2*randomBindingBytes))), withClockForTest(func() time.Time { return fixedNow }), ) var progress bytes.Buffer - callbackResult := make(chan error, 1) + var callbackSecret string credential, err := manager.Login(context.Background(), LoginOptions{ - Timeout: 2 * time.Second, Progress: &progress, OpenURL: func(rawURL string) error { - loginURL, err := url.Parse(rawURL) - if err != nil { - return err - } - if loginURL.Scheme != "https" || loginURL.Host != strings.TrimPrefix(server.URL, "https://") { - t.Errorf("login URL origin = %s://%s", loginURL.Scheme, loginURL.Host) + loginURL, parseErr := url.Parse(rawURL) + if parseErr != nil { + return parseErr } - if loginURL.Query().Get("ppe_env") != "" { - t.Error("login URL carried PPE lane") + if loginURL.Path != loginPagePath || loginURL.Query().Get("force") != "" { + return fmt.Errorf("unexpected login URL") } - payload := loginGrantPayload{ - Type: "login_grant", - Grant: "grant-value", - RandomSecretKey: loginURL.Query().Get("random_secret_key"), + callbackSecret = loginURL.Query().Get("random_secret_key") + payload := accessKeyPayload{ + Type: "access_key", + AccessKey: "page-issued-ak", + UID: "user-100", + TokenID: "ak-id-100", + ExpiredAt: fixedNow.Add(time.Hour).Unix(), + RandomSecretKey: callbackSecret, Source: loginSource, CallbackURL: loginURL.Query().Get("callback"), } - go func() { - status, err := sendCallback(payload.CallbackURL, server.URL, payload) - if err == nil && status != http.StatusOK { - err = errors.New("callback did not return HTTP 200") - } - callbackResult <- err - }() + if status := sendCallback(t, payload.CallbackURL, config.DefaultBaseURL, payload); status != http.StatusOK { + return fmt.Errorf("callback status %d", status) + } return nil }, }) if err != nil { t.Fatal(err) } - if err := <-callbackResult; err != nil { - t.Fatal(err) + if credential.AccessKey != "page-issued-ak" || credential.UID != "user-100" || credential.TokenID != "ak-id-100" || + credential.CredentialScope != credentialScope("user-100", credential.DeviceID) { + t.Fatalf("credential = %#v", credentialWithoutSecret(credential)) } - if generated { - t.Fatal("generate endpoint was called despite exact valid token") + store.mu.Lock() + stored := cloneCredential(store.credential) + store.mu.Unlock() + if stored == nil || stored.AccessKey != credential.AccessKey { + t.Fatalf("stored credential = %#v", credentialWithoutSecret(stored)) } - if credential.AccessKey != "managed-ak" || credential.TokenID != "managed-id" || credential.UID != "123" { - t.Fatalf("credential metadata = %#v", credentialWithoutSecret(credential)) + for _, secret := range []string{credential.AccessKey, callbackSecret} { + if strings.Contains(progress.String(), secret) { + t.Fatalf("progress leaked a secret: %q", progress.String()) + } } - if credential.TokenName != expectedName || len(credential.TokenName) > 48 { - t.Fatalf("token name = %q", credential.TokenName) +} + +func TestManagerReauthenticationPinsAccountAndRequiresRotation(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + old, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{0x33}, deviceIDBytes))) + old.AccessKey = "rejected-ak" + old.TokenID = "rejected-id" + old.UID = "account-a" + old.ExpiredAt = fixedNow.Add(time.Hour).Unix() + old.CredentialScope = credentialScope(old.UID, old.DeviceID) + + t.Run("different account is not saved", func(t *testing.T) { + store := &memoryCredentialStore{credential: cloneCredential(old)} + manager := NewManager(config.Load(), WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) + _, err := manager.Login(context.Background(), LoginOptions{ + ForceRefresh: true, + ExpectedCredentialScope: old.CredentialScope, + OpenURL: callbackOpener(t, fixedNow.Add(time.Hour), "account-b", "new-id", "new-ak", true), + }) + if !errors.Is(err, ErrCredentialAccountMismatch) { + t.Fatalf("mismatch error = %v", err) + } + if store.credential.AccessKey != old.AccessKey || store.saves != 0 { + t.Fatal("account mismatch overwrote the stored credential") + } + }) + + t.Run("same rejected token is refused", func(t *testing.T) { + store := &memoryCredentialStore{credential: cloneCredential(old)} + manager := NewManager(config.Load(), WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) + _, err := manager.Login(context.Background(), LoginOptions{ + ForceRefresh: true, + ExpectedCredentialScope: old.CredentialScope, + OpenURL: callbackOpener(t, fixedNow.Add(time.Hour), old.UID, old.TokenID, old.AccessKey, true), + }) + if err == nil || !strings.Contains(err.Error(), "未轮换") { + t.Fatalf("same-token force login error = %v", err) + } + if strings.Contains(err.Error(), old.AccessKey) { + t.Fatal("force-login error leaked the rejected Access Key") + } + }) + + t.Run("rotated token is accepted", func(t *testing.T) { + store := &memoryCredentialStore{credential: cloneCredential(old)} + manager := NewManager(config.Load(), WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) + got, err := manager.Login(context.Background(), LoginOptions{ + ForceRefresh: true, + ExpectedCredentialScope: old.CredentialScope, + OpenURL: callbackOpener(t, fixedNow.Add(time.Hour), old.UID, "rotated-id", "rotated-ak", true), + }) + if err != nil || got.AccessKey != "rotated-ak" || got.CredentialScope != old.CredentialScope { + t.Fatalf("rotated credential = %#v, %v", credentialWithoutSecret(got), err) + } + }) +} + +func TestBrowserFlowRequiresExpectedAccountForForceAndNeverUsesRawUID(t *testing.T) { + authURL, _ := url.Parse(config.DefaultBaseURL) + deviceID, _ := randomEncoded(bytes.NewReader(bytes.Repeat([]byte{0x22}, deviceIDBytes)), deviceIDBytes) + if _, err := startBrowserFlow( + authURL, bytes.NewReader(bytes.Repeat([]byte{0x42}, 2*randomBindingBytes)), + deviceID, "saved-token-id", "", true, + ); err == nil || !strings.Contains(err.Error(), "登录账号") { + t.Fatalf("force flow without account binding error = %v", err) } - publicJSON, err := json.Marshal(credential) + + const rawUID = "raw-user-id-must-not-be-in-url" + flow, err := startBrowserFlow( + authURL, bytes.NewReader(bytes.Repeat([]byte{0x43}, 2*randomBindingBytes)), + deviceID, "saved-token-id", accountBinding(rawUID), true, + ) if err != nil { t.Fatal(err) } - if strings.Contains(string(publicJSON), credential.AccessKey) { - t.Fatal("credential JSON exposed the Access Key") + defer flow.close() + if strings.Contains(flow.loginURL, rawUID) { + t.Fatal("login URL exposed raw UID") } - if strings.Contains(progress.String(), "managed-ak") || strings.Contains(progress.String(), "grant-value") || strings.Contains(progress.String(), "cookie-secret") { - t.Fatalf("progress leaked credentials: %q", progress.String()) + parsed, err := url.Parse(flow.loginURL) + if err != nil { + t.Fatal(err) + } + if got := parsed.Query().Get("expected_account"); got != accountBinding(rawUID) || !validAccountBinding(got) { + t.Fatalf("expected_account = %q", got) } } -func TestExchangeGeneratesWhenExactValidTokenDoesNotExist(t *testing.T) { - fixedNow := time.Unix(1_800_000_000, 0) - identity, err := newIdentity(bytes.NewReader(bytes.Repeat([]byte{7}, deviceIDBytes))) +func TestBrowserFlowSendsStoredTokenIDOnNormalLogin(t *testing.T) { + authURL, _ := url.Parse(config.DefaultBaseURL) + deviceID, _ := randomEncoded(bytes.NewReader(bytes.Repeat([]byte{0x23}, deviceIDBytes)), deviceIDBytes) + flow, err := startBrowserFlow( + authURL, bytes.NewReader(bytes.Repeat([]byte{0x44}, 2*randomBindingBytes)), + deviceID, "existing-token-id", accountBinding("existing-user"), false, + ) if err != nil { t.Fatal(err) } - server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - assertAuthHeaders(t, request) - switch request.URL.Path { - case exchangeGrantPath: - http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) - writeEnvelope(writer, map[string]any{"uid": "456", "scope": loginGrantScope}) - case queryAccessKeyPath: - writeEnvelope(writer, map[string]any{"access_token_list": []map[string]any{ - {"ak_id": "foreign-id", "token": "foreign-ak", "expired_at": fixedNow.Add(time.Hour).Unix(), "token_name": "foreign", "token_status": "enable"}, - {"ak_id": "expired-id", "token": "expired-ak", "expired_at": fixedNow.Add(-time.Hour).Unix(), "token_name": identity.TokenName, "token_status": "enable"}, - }}) - case generateAccessKeyPath: - var body struct { - TokenName string `json:"token_name"` - ExpiredAt int64 `json:"expired_at"` - } - if err := json.NewDecoder(request.Body).Decode(&body); err != nil { - t.Error(err) - } - if body.TokenName != identity.TokenName { - t.Errorf("generated token name = %q", body.TokenName) - } - if body.ExpiredAt != fixedNow.Add(DefaultCredentialLifetime).Unix() { - t.Errorf("generated expiry = %d", body.ExpiredAt) - } - writeEnvelope(writer, map[string]any{"ak": "new-managed-ak", "token_id": "new-managed-id"}) - default: - http.NotFound(writer, request) - } - })) - defer server.Close() - manager := NewManager(config.Load(), - WithCredentialStore(&memoryCredentialStore{}), - WithHTTPClient(server.Client()), - withAuthBaseURLForTest(server.URL), - withClockForTest(func() time.Time { return fixedNow }), - ) - credential, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ - Grant: "grant", RandomSecretKey: "secret", ExpireAt: fixedNow.Add(time.Minute).Unix(), - }, identity, LoginOptions{}) + defer flow.close() + parsed, err := url.Parse(flow.loginURL) if err != nil { t.Fatal(err) } - if credential.AccessKey != "new-managed-ak" || credential.TokenID != "new-managed-id" || credential.UID != "456" { - t.Fatalf("credential metadata = %#v", credentialWithoutSecret(credential)) + if parsed.Query().Get("token_id") != "existing-token-id" || parsed.Query().Get("force") != "" { + t.Fatalf("normal login query = %v", parsed.Query()) } } -func TestForceRefreshDeletesExactDeviceTokensBeforeGeneratingReplacement(t *testing.T) { - fixedNow := time.Unix(1_800_000_000, 0) - identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{8}, deviceIDBytes))) - identity.AccessKey = "rejected-ak" - identity.TokenID = "rejected-id" - identity.UID = "456" - identity.CredentialScope = credentialScope(identity.UID, identity.DeviceID) - identity.ExpiredAt = fixedNow.Add(time.Hour).Unix() - - var calls []string - server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - calls = append(calls, request.URL.Path) - switch request.URL.Path { - case exchangeGrantPath: - http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) - writeEnvelope(writer, map[string]any{"uid": identity.UID, "scope": loginGrantScope}) - case queryAccessKeyPath: - writeEnvelope(writer, map[string]any{"access_token_list": []map[string]any{ - {"ak_id": identity.TokenID, "token": identity.AccessKey, "expired_at": identity.ExpiredAt, "token_name": "user-renamed-token", "token_status": "enable"}, - {"ak_id": "stale-duplicate", "token": "stale-ak", "expired_at": identity.ExpiredAt, "token_name": identity.TokenName, "token_status": "disable"}, - {"ak_id": "foreign-id", "token": "foreign-ak", "expired_at": identity.ExpiredAt, "token_name": "another-device", "token_status": "enable"}, - }}) - case deleteAccessKeyPath: - var body struct { - AKIDs []string `json:"ak_ids"` - } - if err := json.NewDecoder(request.Body).Decode(&body); err != nil { - t.Error(err) - } - if strings.Join(body.AKIDs, ",") != "rejected-id" { - t.Errorf("deleted IDs = %v", body.AKIDs) - } - writeEnvelope(writer, map[string]any{}) - case generateAccessKeyPath: - writeEnvelope(writer, map[string]any{"ak": "replacement-ak", "token_id": "replacement-id"}) - default: - http.NotFound(writer, request) - } - })) - defer server.Close() - - manager := NewManager(config.Load(), WithHTTPClient(server.Client()), withAuthBaseURLForTest(server.URL), withClockForTest(func() time.Time { return fixedNow })) - credential, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ - Grant: "grant", RandomSecretKey: "secret", ExpireAt: fixedNow.Add(time.Minute).Unix(), - }, identity, LoginOptions{ForceRefresh: true, ExpectedCredentialScope: identity.CredentialScope}) +func TestBrowserFlowAllowsLogoutIdentityTokenWithoutAccountBinding(t *testing.T) { + authURL, _ := url.Parse(config.DefaultBaseURL) + deviceID, _ := randomEncoded(bytes.NewReader(bytes.Repeat([]byte{0x24}, deviceIDBytes)), deviceIDBytes) + flow, err := startBrowserFlow( + authURL, bytes.NewReader(bytes.Repeat([]byte{0x45}, 2*randomBindingBytes)), + deviceID, "existing-token-id", "", false, + ) if err != nil { t.Fatal(err) } - if credential.AccessKey != "replacement-ak" || credential.TokenID != "replacement-id" { - t.Fatalf("replacement credential = %#v", credentialWithoutSecret(credential)) + defer flow.close() + parsed, err := url.Parse(flow.loginURL) + if err != nil { + t.Fatal(err) } - if got := strings.Join(calls, ","); got != exchangeGrantPath+","+queryAccessKeyPath+","+deleteAccessKeyPath+","+generateAccessKeyPath { - t.Fatalf("endpoint order = %q", got) + if parsed.Query().Get("token_id") != "existing-token-id" || parsed.Query().Get("expected_account") != "" || + parsed.Query().Get("force") != "" { + t.Fatalf("logout relogin query = %v", parsed.Query()) } } -func TestForceRefreshRejectsDifferentBrowserAccountBeforeAKMutation(t *testing.T) { +func TestManagerRejectsExpiredPageCredential(t *testing.T) { fixedNow := time.Unix(1_800_000_000, 0) - identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{6}, deviceIDBytes))) - identity.AccessKey = "account-a-ak" - identity.TokenID = "account-a-id" - identity.UID = "account-a" - identity.CredentialScope = credentialScope(identity.UID, identity.DeviceID) - identity.ExpiredAt = fixedNow.Add(time.Hour).Unix() - mutated := false - server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if request.URL.Path != exchangeGrantPath { - mutated = true - http.Error(writer, "unexpected", http.StatusInternalServerError) - return - } - http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) - writeEnvelope(writer, map[string]any{"uid": "account-b", "scope": loginGrantScope}) - })) - defer server.Close() - manager := NewManager(config.Load(), WithHTTPClient(server.Client()), withAuthBaseURLForTest(server.URL), withClockForTest(func() time.Time { return fixedNow })) - _, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ - Grant: "grant", RandomSecretKey: "secret", ExpireAt: fixedNow.Add(time.Minute).Unix(), - }, identity, LoginOptions{ForceRefresh: true, ExpectedCredentialScope: identity.CredentialScope}) - if !errors.Is(err, ErrCredentialAccountMismatch) || mutated { - t.Fatalf("error/mutated = %v/%v, want account mismatch before AK mutation", err, mutated) + store := &memoryCredentialStore{} + manager := NewManager(config.Load(), WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) + _, err := manager.Login(context.Background(), LoginOptions{ + OpenURL: callbackOpener(t, fixedNow.Add(-time.Hour), "user", "id", "ak", false), + }) + if !errors.Is(err, ErrCredentialExpired) { + t.Fatalf("expired callback error = %v", err) } -} - -func TestForceRefreshNeverDeletesStoredTokenAfterAccountSwitch(t *testing.T) { - fixedNow := time.Unix(1_800_000_000, 0) - identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{6}, deviceIDBytes))) - identity.AccessKey = "account-a-ak" - identity.TokenID = "shared-looking-id" - identity.UID = "account-a" - identity.CredentialScope = credentialScope(identity.UID, identity.DeviceID) - identity.ExpiredAt = fixedNow.Add(time.Hour).Unix() - deleteCalled := false - server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - switch request.URL.Path { - case exchangeGrantPath: - http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) - writeEnvelope(writer, map[string]any{"uid": "account-b", "scope": loginGrantScope}) - case queryAccessKeyPath: - writeEnvelope(writer, map[string]any{"access_token_list": []map[string]any{{ - "ak_id": identity.TokenID, "token": "account-b-ak", "expired_at": identity.ExpiredAt, - "token_name": identity.TokenName, "token_status": "enable", - }}}) - case deleteAccessKeyPath: - deleteCalled = true - http.Error(writer, "must not delete", http.StatusInternalServerError) - case generateAccessKeyPath: - writeEnvelope(writer, map[string]any{"ak": "account-b-replacement", "token_id": "account-b-id"}) - default: - http.NotFound(writer, request) - } - })) - defer server.Close() - manager := NewManager(config.Load(), WithHTTPClient(server.Client()), withAuthBaseURLForTest(server.URL), withClockForTest(func() time.Time { return fixedNow })) - credential, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ - Grant: "grant", RandomSecretKey: "secret", ExpireAt: fixedNow.Add(time.Minute).Unix(), - }, identity, LoginOptions{ForceRefresh: true}) - if err != nil || deleteCalled { - t.Fatalf("force refresh after account switch = %#v/%v, deleteCalled=%v", credentialWithoutSecret(credential), err, deleteCalled) - } - if credential.UID != "account-b" || credential.TokenID != "account-b-id" { - t.Fatalf("replacement credential = %#v", credentialWithoutSecret(credential)) + // The fresh device identity is safe to retain, but the expired AK must not be saved. + if store.credential == nil || store.credential.AccessKey != "" { + t.Fatalf("expired callback was saved: %#v", credentialWithoutSecret(store.credential)) } } -func TestGenerateAccessKeyPermissionAndLimitGuidance(t *testing.T) { - for _, test := range []struct { - name string - ret string - message string - }{ - {name: "permission", ret: "3", message: "暂不具备创建 CLI Access Key 的权限"}, - {name: "possible limit", ret: "12001", message: "Access Key 数量上限"}, - } { - t.Run(test.name, func(t *testing.T) { - fixedNow := time.Unix(1_800_000_000, 0) - identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{5}, deviceIDBytes))) - server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - switch request.URL.Path { - case exchangeGrantPath: - http.SetCookie(writer, &http.Cookie{Name: "session", Value: "session-value", Path: "/", Secure: true}) - writeEnvelope(writer, map[string]any{"uid": "123", "scope": loginGrantScope}) - case queryAccessKeyPath: - writeEnvelope(writer, map[string]any{"access_token_list": []any{}}) - case generateAccessKeyPath: - _ = json.NewEncoder(writer).Encode(map[string]any{"ret": test.ret, "errmsg": "unsafe upstream detail"}) - default: - http.NotFound(writer, request) - } - })) - defer server.Close() - manager := NewManager(config.Load(), WithHTTPClient(server.Client()), withAuthBaseURLForTest(server.URL), withClockForTest(func() time.Time { return fixedNow })) - _, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{Grant: "grant", RandomSecretKey: "secret"}, identity, LoginOptions{}) - if err == nil || !strings.Contains(err.Error(), test.message) || strings.Contains(err.Error(), "unsafe upstream detail") { - t.Fatalf("error = %v, want safe guidance containing %q", err, test.message) +func TestManagerDerivesExpectedAccountFromDurableScope(t *testing.T) { + fixedNow := time.Unix(1_800_000_000, 0) + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{0x25}, deviceIDBytes))) + identity.TokenID = "saved-token-id" + store := &memoryCredentialStore{credential: identity} + manager := NewManager(config.Load(), WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) + expectedScope := credentialScope("durable-user", identity.DeviceID) + _, err := manager.Login(context.Background(), LoginOptions{ + ForceRefresh: true, + ExpectedCredentialScope: expectedScope, + OpenURL: func(rawURL string) error { + loginURL, parseErr := url.Parse(rawURL) + if parseErr != nil { + return parseErr } - }) - } -} - -func TestCredentialScopeBindsUIDAndDeviceButNotAK(t *testing.T) { - device := base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{1}, deviceIDBytes)) - first := credentialScope("account-a", device) - if first != credentialScope("account-a", device) { - t.Fatal("scope changed for the same UID and device") - } - if first == credentialScope("account-b", device) { - t.Fatal("scope did not change across accounts") - } - otherDevice := base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{2}, deviceIDBytes)) - if first == credentialScope("account-a", otherDevice) { - t.Fatal("scope did not change across devices") + if got := loginURL.Query().Get("expected_account"); got != accountBinding("durable-user") || + loginURL.Query().Get("force") != "1" || loginURL.Query().Get("token_id") != identity.TokenID || + strings.Contains(rawURL, "durable-user") { + return fmt.Errorf("durable reauth binding is incomplete") + } + payload := accessKeyPayload{ + Type: "access_key", + AccessKey: "new-ak", + UID: "durable-user", + TokenID: "rotated-token-id", + ExpiredAt: fixedNow.Add(time.Hour).Unix(), + RandomSecretKey: loginURL.Query().Get("random_secret_key"), + Source: loginSource, + CallbackURL: loginURL.Query().Get("callback"), + } + if status := sendCallback(t, payload.CallbackURL, config.DefaultBaseURL, payload); status != http.StatusOK { + return fmt.Errorf("callback status %d", status) + } + return nil + }, + }) + if err != nil { + t.Fatal(err) } } -func TestSelectManagedTokenRejectsAmbiguousExactNames(t *testing.T) { +func TestResolveStatusLogoutAndAuthOrigin(t *testing.T) { fixedNow := time.Unix(1_800_000_000, 0) - identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{9}, deviceIDBytes))) - manager := NewManager(config.Load(), withClockForTest(func() time.Time { return fixedNow })) - tokens := []accessToken{ - {ID: "one", Token: "ak-one", Name: identity.TokenName, Status: "enable", ExpiredAt: fixedNow.Add(time.Hour).Unix()}, - {ID: "two", Token: "ak-two", Name: identity.TokenName, Status: "enable", ExpiredAt: fixedNow.Add(time.Hour).Unix()}, + credential, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{3}, deviceIDBytes))) + credential.AccessKey = "stored-ak" + credential.TokenID = "stored-id" + credential.UID = "789" + credential.CredentialScope = credentialScope(credential.UID, credential.DeviceID) + credential.ExpiredAt = fixedNow.Add(time.Hour).Unix() + store := &memoryCredentialStore{credential: credential} + cfg := config.Load() + cfg.AccessKey = " env-ak " + cfg.BaseURL = "https://untrusted.invalid" + cfg.PPEEnv = "ppe_untrusted" + manager := NewManager(cfg, WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) + if manager.authBaseURL.String() != config.DefaultBaseURL { + t.Fatalf("browser login followed runtime PPE/base URL: %q", manager.authBaseURL) } - if _, err := manager.selectManagedToken(tokens, identity); err == nil { - t.Fatal("ambiguous exact device tokens were accepted") + if key, err := manager.ResolveAccessKey(context.Background()); err != nil || key != "env-ak" { + t.Fatalf("environment precedence = %q, %v", key, err) } - identity.TokenID = "two" - selected, err := manager.selectManagedToken(tokens, identity) - if err != nil { - t.Fatal(err) + status, err := manager.Status(context.Background()) + if err != nil || !status.LoggedIn || status.Source != "environment" { + t.Fatalf("environment status = %#v, %v", status, err) } - if selected.ID != "two" { - t.Fatalf("selected ID = %q", selected.ID) + + cfg.AccessKey = "" + if key, err := manager.ResolveAccessKey(context.Background()); err != nil || key != credential.AccessKey { + t.Fatalf("stored resolution = %q, %v", key, err) } - selected, err = manager.selectManagedToken([]accessToken{ - {ID: "two", Token: "renamed-ak", Name: "renamed-by-user", Status: "enable", ExpiredAt: fixedNow.Add(time.Hour).Unix()}, - }, identity) - if err != nil || selected == nil || selected.Token != "renamed-ak" { - t.Fatalf("renamed exact TokenID was not reused: %#v/%v", selected, err) + if scope, err := manager.CredentialScope(context.Background()); err != nil || scope != credential.CredentialScope { + t.Fatalf("credential scope = %q, %v", scope, err) } -} - -func TestAuthErrorsDoNotLeakRequestOrResponseSecrets(t *testing.T) { - const ( - grant = "grant-must-stay-secret" - secret = "binding-must-stay-secret" - cookie = "cookie-must-stay-secret" - ) - server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - http.SetCookie(writer, &http.Cookie{Name: "session", Value: cookie, Secure: true}) - writer.WriteHeader(http.StatusInternalServerError) - _, _ = writer.Write([]byte(grant + secret + cookie)) - })) - defer server.Close() - identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{2}, deviceIDBytes))) - manager := NewManager(config.Load(), - WithCredentialStore(&memoryCredentialStore{}), - WithHTTPClient(server.Client()), - withAuthBaseURLForTest(server.URL), - ) - _, err := manager.exchangeAndProvision(context.Background(), loginGrantPayload{ - Grant: grant, RandomSecretKey: secret, ExpireAt: time.Now().Add(time.Minute).Unix(), - }, identity, LoginOptions{}) - if err == nil { - t.Fatal("exchange error = nil") - } - for _, value := range []string{grant, secret, cookie} { - if strings.Contains(err.Error(), value) { - t.Fatalf("error leaked secret: %q", err) - } + if err := manager.Logout(context.Background(), true); !errors.Is(err, ErrRemoteRevokeUnsupported) { + t.Fatalf("remote revoke error = %v", err) } -} - -func TestResilientStoreFallsBackOnlyToSecureStore(t *testing.T) { - identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{4}, deviceIDBytes))) - primary := &memoryCredentialStore{loadErr: ErrSecureStore, saveErr: ErrSecureStore, deleteErr: ErrSecureStore} - fallback := &memoryCredentialStore{} - store := &resilientCredentialStore{primary: primary, fallback: fallback} - if err := store.Save(context.Background(), identity); err != nil { + if err := manager.Logout(context.Background(), false); err != nil { t.Fatal(err) } - loaded, err := store.Load(context.Background()) - if err != nil || loaded.DeviceID != identity.DeviceID { - t.Fatalf("fallback load = %#v, %v", loaded, err) + if store.credential == nil || store.credential.DeviceID != credential.DeviceID || store.credential.AccessKey != "" || + store.credential.UID != "" || store.credential.TokenID != credential.TokenID { + t.Fatalf("logout did not retain only device identity: %#v", credentialWithoutSecret(store.credential)) } - if err := store.Delete(context.Background()); !errors.Is(err, ErrSecureStore) { - t.Fatalf("delete must report unresolved primary keyring failure, got %v", err) + if _, err := manager.ResolveAccessKey(context.Background()); !errors.Is(err, ErrCredentialNotFound) { + t.Fatalf("post-logout resolution = %v", err) } } -func TestManagerFreshIdentityUsesEmptyFallbackWhenKeyringUnavailable(t *testing.T) { - primary := &memoryCredentialStore{loadErr: ErrSecureStore, saveErr: ErrSecureStore} - fallback := &memoryCredentialStore{} - store := &resilientCredentialStore{primary: primary, fallback: fallback} - manager := NewManager( - config.Load(), - WithCredentialStore(store), - withRandomReaderForTest(bytes.NewReader(bytes.Repeat([]byte{0x2a}, deviceIDBytes))), - ) - - identity, err := manager.ensureIdentity(context.Background()) +func TestDecodeCredentialAcceptsLegacyTokenName(t *testing.T) { + deviceID, _ := randomEncoded(bytes.NewReader(bytes.Repeat([]byte{0x44}, deviceIDBytes)), deviceIDBytes) + uid := "legacy-user" + payload, err := json.Marshal(map[string]any{ + "version": credentialVersion, + "device_id": deviceID, + "credential_scope": credentialScope(uid, deviceID), + "token_name": "pippit-tool-cli-legacy-name", + "access_key": "legacy-ak", + "token_id": "legacy-id", + "uid": uid, + "expired_at": time.Now().Add(time.Hour).Unix(), + }) if err != nil { - t.Fatalf("ensureIdentity() error = %v", err) - } - if identity.AccessKey != "" || !validDeviceID(identity.DeviceID) { - t.Fatalf("fresh identity = %#v", credentialWithoutSecret(identity)) + t.Fatal(err) } - fallback.mu.Lock() - stored := cloneCredential(fallback.credential) - fallbackLoads, fallbackSaves := fallback.loads, fallback.saves - fallback.mu.Unlock() - if stored == nil || stored.DeviceID != identity.DeviceID || fallbackLoads != 1 || fallbackSaves != 1 { - t.Fatalf("fallback identity/loads/saves = %#v/%d/%d", credentialWithoutSecret(stored), fallbackLoads, fallbackSaves) + credential, err := decodeCredential(payload) + if err != nil || credential.AccessKey != "legacy-ak" || credential.DeviceID != deviceID { + t.Fatalf("legacy credential = %#v, %v", credentialWithoutSecret(credential), err) } - if primary.loads != 1 || primary.saves != 1 { - t.Fatalf("primary loads/saves = %d/%d, want 1/1", primary.loads, primary.saves) + encoded, err := encodeCredential(credential) + if err != nil { + t.Fatal(err) } - - again, err := manager.ensureIdentity(context.Background()) - if err != nil || again.DeviceID != identity.DeviceID || fallback.loads != 1 { - t.Fatalf("cached ensureIdentity() = %#v/%v, fallback loads=%d", credentialWithoutSecret(again), err, fallback.loads) + if bytes.Contains(encoded, []byte("token_name")) { + t.Fatalf("new credential encoding retained legacy field: %s", encoded) } } -func TestResilientStoreDoesNotMaskCorruptPrimaryOrCleanupFailure(t *testing.T) { - identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{4}, deviceIDBytes))) - corrupt := errors.New("corrupt primary credential") - primary := &memoryCredentialStore{loadErr: corrupt} - fallback := &memoryCredentialStore{credential: identity} - store := &resilientCredentialStore{primary: primary, fallback: fallback} - if _, err := store.Load(context.Background()); !errors.Is(err, corrupt) { - t.Fatalf("corrupt primary was masked by fallback: %v", err) - } - if fallback.loads != 0 { - t.Fatalf("fallback loads = %d, want zero after corrupt primary", fallback.loads) +func TestManagerNormalizesLegacyDeviceScopeOnLoad(t *testing.T) { + credential, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{0x45}, deviceIDBytes))) + credential.AccessKey = "legacy-ak" + credential.TokenID = "legacy-id" + credential.UID = "legacy-user" + credential.ExpiredAt = time.Now().Add(time.Hour).Unix() + credential.CredentialScope = legacyCredentialScope(credential.DeviceID) + payload, err := encodeCredential(credential) + if err != nil { + t.Fatal(err) } - - primary = &memoryCredentialStore{} - fallback = &memoryCredentialStore{credential: identity, deleteErr: ErrSecureStore} - store = &resilientCredentialStore{primary: primary, fallback: fallback} - if err := store.Save(context.Background(), identity); !errors.Is(err, ErrSecureStore) { - t.Fatalf("stale fallback cleanup error was swallowed: %v", err) + decoded, err := decodeCredential(payload) + if err != nil { + t.Fatal(err) } - if primary.saves != 1 || fallback.deletes != 1 { - t.Fatalf("primary saves/fallback deletes = %d/%d", primary.saves, fallback.deletes) + store := &memoryCredentialStore{credential: decoded} + manager := NewManager(config.Load(), WithCredentialStore(store)) + got, err := manager.loadCredential(context.Background()) + if err != nil { + t.Fatal(err) } - - primary = &memoryCredentialStore{saveErr: errors.New("invalid primary write")} - fallback = &memoryCredentialStore{} - store = &resilientCredentialStore{primary: primary, fallback: fallback} - if err := store.Save(context.Background(), identity); err == nil || fallback.saves != 0 { - t.Fatalf("non-secure-store primary failure fell back: err=%v fallback saves=%d", err, fallback.saves) + if got.CredentialScope != credentialScope(credential.UID, credential.DeviceID) { + t.Fatalf("legacy scope was not normalized: %q", got.CredentialScope) } } -func TestManagerCachesCredentialAndLogoutPreservesDeviceIdentity(t *testing.T) { +func TestManagerCredentialCacheIsConcurrent(t *testing.T) { fixedNow := time.Unix(1_800_000_000, 0) credential, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{3}, deviceIDBytes))) credential.AccessKey = "stored-ak" @@ -659,9 +519,9 @@ func TestManagerCachesCredentialAndLogoutPreservesDeviceIdentity(t *testing.T) { cfg.AccessKey = "" manager := NewManager(cfg, WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) - const callers = 32 + const callers = 24 start := make(chan struct{}) - errs := make(chan error, callers) + errCh := make(chan error, callers) var group sync.WaitGroup for index := 0; index < callers; index++ { group.Add(1) @@ -670,119 +530,53 @@ func TestManagerCachesCredentialAndLogoutPreservesDeviceIdentity(t *testing.T) { <-start key, err := manager.ResolveAccessKey(context.Background()) if err == nil && key != credential.AccessKey { - err = fmt.Errorf("key = %q", key) + err = fmt.Errorf("unexpected access key") } - errs <- err + errCh <- err }() } close(start) group.Wait() - close(errs) - for err := range errs { + close(errCh) + for err := range errCh { if err != nil { t.Fatal(err) } } if store.loads != 1 { - t.Fatalf("keyring loads = %d, want one process-local load", store.loads) - } - - replacement := cloneCredential(credential) - replacement.AccessKey = "replacement-ak" - replacement.TokenID = "replacement-id" - if err := manager.saveCredential(context.Background(), replacement); err != nil { - t.Fatal(err) - } - if key, err := manager.ResolveAccessKey(context.Background()); err != nil || key != replacement.AccessKey || store.loads != 1 { - t.Fatalf("cached replacement = %q/%v, loads=%d", key, err, store.loads) - } - - if err := manager.Logout(context.Background(), false); err != nil { - t.Fatal(err) - } - store.mu.Lock() - preserved := cloneCredential(store.credential) - deletes := store.deletes - store.mu.Unlock() - if preserved.DeviceID != credential.DeviceID || preserved.TokenName != credential.TokenName || - preserved.AccessKey != "" || preserved.TokenID != replacement.TokenID || preserved.UID != "" || preserved.CredentialScope != "" { - t.Fatalf("logout did not retain only reusable non-secret identity: %#v", credentialWithoutSecret(preserved)) - } - if deletes != 0 { - t.Fatalf("logout deleted the device identity %d time(s)", deletes) - } - if _, err := manager.ResolveAccessKey(context.Background()); !errors.Is(err, ErrCredentialNotFound) { - t.Fatalf("post-logout resolution = %v", err) - } - if store.loads != 2 { - t.Fatalf("post-logout cache was not cleared; loads=%d", store.loads) - } - selected, err := manager.selectManagedToken([]accessToken{{ - ID: replacement.TokenID, Token: replacement.AccessKey, Name: preserved.TokenName, - Status: "enable", ExpiredAt: replacement.ExpiredAt, - }}, preserved) - if err != nil || selected == nil || selected.ID != replacement.TokenID { - t.Fatalf("preserved identity could not reuse remote token: %#v/%v", selected, err) + t.Fatalf("credential store loads = %d, want 1", store.loads) } } -func TestManagerAuthOriginIgnoresRuntimeBaseURLAndPPE(t *testing.T) { - cfg := config.Load() - cfg.BaseURL = "https://untrusted.invalid" - cfg.PPEEnv = "ppe_untrusted" - manager := NewManager(cfg, WithCredentialStore(&memoryCredentialStore{})) - if got := manager.authBaseURL.String(); got != config.DefaultBaseURL { - t.Fatalf("auth base URL = %q, want %q", got, config.DefaultBaseURL) - } -} - -func TestResolveAccessKeyPrecedenceStatusAndScope(t *testing.T) { - fixedNow := time.Unix(1_800_000_000, 0) - identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{3}, deviceIDBytes))) - identity.AccessKey = "stored-ak" - identity.TokenID = "stored-id" - identity.UID = "789" - identity.CredentialScope = credentialScope(identity.UID, identity.DeviceID) - identity.ExpiredAt = fixedNow.Add(time.Hour).Unix() - store := &memoryCredentialStore{credential: identity} - cfg := config.Load() - cfg.AccessKey = " env-ak " - manager := NewManager(cfg, WithCredentialStore(store), withClockForTest(func() time.Time { return fixedNow })) - accessKey, err := manager.ResolveAccessKey(context.Background()) - if err != nil || accessKey != "env-ak" { - t.Fatalf("explicit resolution = %q, %v", accessKey, err) - } - status, err := manager.Status(context.Background()) - if err != nil || !status.LoggedIn || status.Source != "environment" { - t.Fatalf("environment status = %#v, %v", status, err) - } - - cfg.AccessKey = "" - accessKey, err = manager.ResolveAccessKey(context.Background()) - if err != nil || accessKey != "stored-ak" { - t.Fatalf("stored resolution = %q, %v", accessKey, err) - } - scope, err := manager.CredentialScope(context.Background()) - if err != nil || scope != identity.CredentialScope { - t.Fatalf("scope = %q, %v", scope, err) - } - expired := cloneCredential(identity) - expired.ExpiredAt = fixedNow.Unix() - if err := manager.saveCredential(context.Background(), expired); err != nil { +func TestResilientStoreFallbackBoundaries(t *testing.T) { + identity, _ := newIdentity(bytes.NewReader(bytes.Repeat([]byte{4}, deviceIDBytes))) + primary := &memoryCredentialStore{loadErr: ErrSecureStore, saveErr: ErrSecureStore, deleteErr: ErrSecureStore} + fallback := &memoryCredentialStore{} + store := &resilientCredentialStore{primary: primary, fallback: fallback} + if err := store.Save(context.Background(), identity); err != nil { t.Fatal(err) } - if _, err := manager.ResolveAccessKey(context.Background()); !errors.Is(err, ErrCredentialExpired) { - t.Fatalf("expired error = %v", err) + loaded, err := store.Load(context.Background()) + if err != nil || loaded.DeviceID != identity.DeviceID { + t.Fatalf("fallback load = %#v, %v", credentialWithoutSecret(loaded), err) } - if err := manager.Logout(context.Background(), true); !errors.Is(err, ErrRemoteRevokeUnsupported) { - t.Fatalf("revoke error = %v", err) + if err := store.Delete(context.Background()); !errors.Is(err, ErrSecureStore) { + t.Fatalf("primary delete failure was hidden: %v", err) } - if err := manager.Logout(context.Background(), false); err != nil { - t.Fatal(err) + + corrupt := errors.New("corrupt primary credential") + primary = &memoryCredentialStore{loadErr: corrupt} + fallback = &memoryCredentialStore{credential: identity} + store = &resilientCredentialStore{primary: primary, fallback: fallback} + if _, err := store.Load(context.Background()); !errors.Is(err, corrupt) || fallback.loads != 0 { + t.Fatalf("corrupt primary was masked: err=%v fallback-loads=%d", err, fallback.loads) } } func TestFileCredentialStoreIsPrivateAtomicAndNoFollow(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows intentionally uses the system keyring without a file fallback") + } directory := filepath.Join(t.TempDir(), "auth") path := filepath.Join(directory, credentialFileName) store := NewFileCredentialStore(path) @@ -796,11 +590,8 @@ func TestFileCredentialStoreIsPrivateAtomicAndNoFollow(t *testing.T) { t.Fatal(err) } info, err := os.Stat(path) - if err != nil { - t.Fatal(err) - } - if info.Mode().Perm() != 0o600 { - t.Fatalf("credential mode = %o", info.Mode().Perm()) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("credential file mode = %v, %v", info, err) } loaded, err := store.Load(context.Background()) if err != nil || loaded.AccessKey != credential.AccessKey { @@ -825,7 +616,7 @@ func TestFileCredentialStoreIsPrivateAtomicAndNoFollow(t *testing.T) { } targetData, _ := os.ReadFile(target) if string(targetData) != "must-not-change" { - t.Fatal("atomic save followed and overwrote symlink target") + t.Fatal("atomic save followed the symlink target") } } @@ -838,8 +629,7 @@ func TestSanitizedBrowserEnv(t *testing.T) { "PIPPIT_CLI_PPE_ENV=ppe_safe", "OTHER_TOKEN=unrelated", } - got := SanitizedBrowserEnv(input) - joined := strings.Join(got, "\n") + joined := strings.Join(SanitizedBrowserEnv(input), "\n") for _, forbidden := range []string{"XYQ_ACCESS_KEY", "PIPPIT_TOKEN", "PIPPIT_CLI_AK"} { if strings.Contains(joined, forbidden) { t.Fatalf("browser env retained %s", forbidden) @@ -852,48 +642,80 @@ func TestSanitizedBrowserEnv(t *testing.T) { } } -func postCallback(t *testing.T, callbackURL, origin string, payload loginGrantPayload) int { +func callbackOpener(t *testing.T, expiry time.Time, uid, tokenID, accessKey string, wantForce bool) func(string) error { t.Helper() - status, err := sendCallback(callbackURL, origin, payload) - if err != nil { - t.Fatal(err) + return func(rawURL string) error { + loginURL, err := url.Parse(rawURL) + if err != nil { + return err + } + if (loginURL.Query().Get("force") == "1") != wantForce { + return fmt.Errorf("force query mismatch") + } + if wantForce && loginURL.Query().Get("token_id") == "" { + return fmt.Errorf("force login omitted token_id") + } + if wantForce && loginURL.Query().Get("expected_account") == "" { + return fmt.Errorf("force login omitted expected_account") + } + payload := accessKeyPayload{ + Type: "access_key", + AccessKey: accessKey, + UID: uid, + TokenID: tokenID, + ExpiredAt: expiry.Unix(), + RandomSecretKey: loginURL.Query().Get("random_secret_key"), + Source: loginSource, + CallbackURL: loginURL.Query().Get("callback"), + } + if status := sendCallback(t, payload.CallbackURL, config.DefaultBaseURL, payload); status != http.StatusOK { + return fmt.Errorf("callback status %d", status) + } + return nil } - return status } -func sendCallback(callbackURL, origin string, payload loginGrantPayload) (int, error) { +func sendCallback(t *testing.T, callbackURL, origin string, payload accessKeyPayload) int { + t.Helper() body, err := json.Marshal(payload) if err != nil { - return 0, err + t.Fatal(err) } - request, err := http.NewRequest(http.MethodPost, callbackURL, bytes.NewReader(body)) + return postRawCallback(t, callbackURL, origin, string(body)) +} + +func postRawCallback(t *testing.T, callbackURL, origin, body string) int { + t.Helper() + request, err := http.NewRequest(http.MethodPost, callbackURL, strings.NewReader(body)) if err != nil { - return 0, err + t.Fatal(err) } request.Header.Set("Origin", origin) request.Header.Set("Content-Type", "application/json") response, err := http.DefaultClient.Do(request) if err != nil { - return 0, err + t.Fatal(err) } + defer response.Body.Close() _, _ = io.Copy(io.Discard, response.Body) - response.Body.Close() - return response.StatusCode, nil -} - -func writeEnvelope(writer http.ResponseWriter, data any) { - writer.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(writer).Encode(map[string]any{"ret": "0", "data": data}) + return response.StatusCode } -func assertAuthHeaders(t *testing.T, request *http.Request) { +func sendPreflight(t *testing.T, callbackURL, origin string) int { t.Helper() - want := map[string]string{"appvr": "1.1.4", "entrance-from": "web", "appid": "795647"} - for name, value := range want { - if got := request.Header.Get(name); got != value { - t.Errorf("header %s = %q, want %q", name, got, value) - } + request, err := http.NewRequest(http.MethodOptions, callbackURL, nil) + if err != nil { + t.Fatal(err) } + request.Header.Set("Origin", origin) + request.Header.Set("Access-Control-Request-Method", "POST") + request.Header.Set("Access-Control-Request-Headers", "Content-Type") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + return response.StatusCode } func credentialWithoutSecret(credential *Credential) any { @@ -901,11 +723,21 @@ func credentialWithoutSecret(credential *Credential) any { return nil } return struct { + Version int DeviceID string CredentialScope string - TokenName string TokenID string UID string ExpiredAt int64 - }{credential.DeviceID, credential.CredentialScope, credential.TokenName, credential.TokenID, credential.UID, credential.ExpiredAt} + }{credential.Version, credential.DeviceID, credential.CredentialScope, credential.TokenID, credential.UID, credential.ExpiredAt} +} + +func credentialPayloadWithoutSecret(payload accessKeyPayload) any { + return struct { + Type string + UID string + TokenID string + ExpiredAt int64 + Source string + }{payload.Type, payload.UID, payload.TokenID, payload.ExpiredAt, payload.Source} } From 0eeeb14e18dabf9b0fef92e36704f2f717cb0005 Mon Sep 17 00:00:00 2001 From: "xuyan.smackgg" <xuyan.smackgg@bytedance.com> Date: Wed, 12 Aug 2026 17:04:29 +0800 Subject: [PATCH 48/48] fix(auth): accept signed browser callback transport params Co-authored-by: Codex <codex@openai.com> --- internal/auth/auth_test.go | 8 ++++++-- internal/auth/loopback.go | 24 +++++++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 85e17d5..e4a2945 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -145,11 +145,15 @@ func TestBrowserFlowUsesDedicatedPageAndStrictBoundCallback(t *testing.T) { if status := sendCallback(t, callbackURL+"&state=duplicate", config.DefaultBaseURL, payload); status != http.StatusBadRequest { t.Fatalf("duplicate state status = %d", status) } + if status := sendCallback(t, callbackURL+"&unexpected=value", config.DefaultBaseURL, payload); status != http.StatusBadRequest { + t.Fatalf("unexpected transport query status = %d", status) + } if status := sendPreflight(t, callbackURL, config.DefaultBaseURL); status != http.StatusNoContent { t.Fatalf("preflight status = %d", status) } - if status := sendCallback(t, callbackURL, config.DefaultBaseURL, payload); status != http.StatusOK { - t.Fatalf("valid callback status = %d", status) + securityRuntimeCallback := callbackURL + "&a_bogus=signed-request&msToken=transport-token" + if status := sendCallback(t, securityRuntimeCallback, config.DefaultBaseURL, payload); status != http.StatusOK { + t.Fatalf("security-runtime callback status = %d", status) } got, err := flow.wait(context.Background()) if err != nil || got.AccessKey != payload.AccessKey || got.UID != payload.UID || got.TokenID != payload.TokenID { diff --git a/internal/auth/loopback.go b/internal/auth/loopback.go index ca64c27..5312ec9 100644 --- a/internal/auth/loopback.go +++ b/internal/auth/loopback.go @@ -18,6 +18,8 @@ import ( const maxCallbackBodyBytes = 64 << 10 +const maxInjectedCallbackQueryValueBytes = 4 << 10 + type accessKeyPayload struct { Type string `json:"type"` AccessKey string `json:"access_key"` @@ -255,7 +257,27 @@ func (f *browserFlow) validRequestTarget(request *http.Request) bool { } query := request.URL.Query() states, ok := query["state"] - return ok && len(query) == 1 && len(states) == 1 && constantTimeEqual(states[0], f.state) + if !ok || len(states) != 1 || !constantTimeEqual(states[0], f.state) { + return false + } + for key, values := range query { + if key == "state" { + continue + } + // The site's security runtime appends these transport-only query + // parameters to cross-origin requests. They are not part of the + // callback binding: state, Origin, callback_url, and the body secret + // remain exact and independently verified below. + if key != "a_bogus" && key != "msToken" || len(values) != 1 || + !validInjectedCallbackQueryValue(values[0]) { + return false + } + } + return true +} + +func validInjectedCallbackQueryValue(value string) bool { + return value != "" && len(value) <= maxInjectedCallbackQueryValueBytes && strings.TrimSpace(value) == value } func (f *browserFlow) callbackURLHost() string {