Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
620 changes: 620 additions & 0 deletions .github/model-e2e/linux/package-lock.json

Large diffs are not rendered by default.

11 changes: 11 additions & 0 deletions .github/model-e2e/linux/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"name": "skill-up-model-e2e-linux-tools",
"private": true,
"version": "0.0.0",
"dependencies": {
"@anthropic-ai/claude-code": "2.1.217",
"@openai/codex": "0.80.0",
"@qwen-code/qwen-code": "0.20.1",
"opencode-ai": "1.18.33"
}
}
31 changes: 7 additions & 24 deletions .github/workflows/model-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,33 +68,16 @@ jobs:
with:
node-version: "22"

- name: Install claude-code CLI
if: steps.secrets.outputs.available == 'true'
run: |
set -euo pipefail
npm install -g --include=optional '@anthropic-ai/claude-code@2.1.217'
claude --version

- name: Install codex CLI
if: steps.secrets.outputs.available == 'true'
run: |
set -euo pipefail
npm install -g --include=optional '@openai/codex@0.80.0'
codex --version

- name: Install OpenCode CLI
- name: Install locked agent CLIs
if: steps.secrets.outputs.available == 'true'
run: |
set -euo pipefail
npm install -g --include=optional --registry=https://registry.npmjs.org 'opencode-ai@1.18.33'
opencode --version

- name: Install qwen-code CLI
if: steps.secrets.outputs.available == 'true'
run: |
set -euo pipefail
npm install -g '@qwen-code/qwen-code@0.20.1'
qwen --version
cd .github/model-e2e/linux
npm ci --include=optional --registry=https://registry.npmjs.org
for cli in claude codex opencode qwen; do
./node_modules/.bin/"$cli" --version
done
echo "$(pwd)/node_modules/.bin" >> "$GITHUB_PATH"

- name: Install qodercli (pinned v1.1.41)
if: steps.secrets.outputs.available == 'true'
Expand Down
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
DeepSeek Harness packages to 0.1.5-rc.3 while retaining the required cordis 4.0.2.

### Fixed
- Use patched Vite/esbuild dependencies for the documentation site while
retaining VitePress 1.x.
- Remove overflow-prone runtime kwargs preallocation while preserving
eval-over-user precedence and independent merged maps.
- Avoid logging API key fragments and redact secrets in custom-engine
output-file paths and errors without changing file operations or fallbacks.
- Expand full-value `$VAR` references in real MCP headers and OpenCode endpoints, matching the existing `${VAR}` form.
- Reject whitespace-only `input.prompt` and `input.turns[].content` during validation while preserving mutual-exclusion semantics and the original prompt bytes.
- Clarify nested YAML comments and keep `skill-upper` scaffolding focused on
Expand Down
8 changes: 4 additions & 4 deletions internal/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -273,18 +273,18 @@ func (a *BaseAgent) CheckCredentials(ctx context.Context) error {
return nil
}

func (a *BaseAgent) logCredentialStatus(ctx context.Context, apiKeyEnv, baseURLEnv, missingMessage string) {
func (a *BaseAgent) logCredentialStatus(ctx context.Context, credentialEnvName, baseURLEnv, missingMessage string) {
if a.Cfg.APIKey != "" {
logging.DebugContextf(ctx, "%s detected for %s (masked: %s, source=agent_config)", apiKeyEnv, a.Name(), credential.MaskAPIKey(a.Cfg.APIKey))
logging.DebugContextf(ctx, "%s detected for %s (source=agent_config)", credentialEnvName, a.Name())
if baseURLEnv != "" && a.Cfg.BaseURL != "" {
logging.DebugContextf(ctx, "%s detected for %s (value: %s, source=agent_config)", baseURLEnv, a.Name(), a.Cfg.BaseURL)
}
return
}
// Fall back to the current provider's process env for observability only.
// This does not scan unrelated providers.
if apiKey := os.Getenv(apiKeyEnv); apiKey != "" {
logging.DebugContextf(ctx, "%s detected for %s (source=process_env)", apiKeyEnv, a.Name())
if os.Getenv(credentialEnvName) != "" {
logging.DebugContextf(ctx, "%s detected for %s (source=process_env)", credentialEnvName, a.Name())
if baseURLEnv != "" && os.Getenv(baseURLEnv) != "" {
logging.DebugContextf(ctx, "%s detected for %s (source=process_env)", baseURLEnv, a.Name())
}
Expand Down
8 changes: 4 additions & 4 deletions internal/agent/claude_code_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (

const testHello = "hello"

func TestClaudeCodeCheckCredentials_LogsMaskedTokenAndBaseURL(t *testing.T) {
func TestClaudeCodeCheckCredentials_LogsTokenPresenceAndBaseURL(t *testing.T) {
// Not parallel: captureStdout redirects package-level log output.

logging.SetVerbosity(1)
Expand All @@ -39,11 +39,11 @@ func TestClaudeCodeCheckCredentials_LogsMaskedTokenAndBaseURL(t *testing.T) {
t.Fatalf("expected no error, got %v", err)
}
})
if !strings.Contains(output, `level=DEBUG msg="ANTHROPIC_API_KEY detected for claude-code (masked: `) {
if !strings.Contains(output, `level=DEBUG msg="ANTHROPIC_API_KEY detected for claude-code (source=agent_config)"`) {
t.Fatalf("expected api key log, got %q", output)
}
if !strings.Contains(output, credential.MaskAPIKey(apiKey)) {
t.Fatalf("expected masked key log, got %q", output)
if strings.Contains(output, apiKey) || strings.Contains(output, "****") {
t.Fatalf("key material leaked into log: %q", output)
}
if !strings.Contains(output, `level=DEBUG msg="ANTHROPIC_BASE_URL detected for claude-code (value: https://anthropic.example.com, source=agent_config)"`) {
t.Fatalf("expected base url log, got %q", output)
Expand Down
8 changes: 4 additions & 4 deletions internal/agent/codex_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func TestCodexCheckCredentials(t *testing.T) {
}
}

func TestCodexCheckCredentials_LogsMaskedTokenAndBaseURL(t *testing.T) {
func TestCodexCheckCredentials_LogsTokenPresenceAndBaseURL(t *testing.T) {
// Not parallel: captureStdout redirects package-level log output.

logging.SetVerbosity(1)
Expand All @@ -132,11 +132,11 @@ func TestCodexCheckCredentials_LogsMaskedTokenAndBaseURL(t *testing.T) {
t.Fatalf("expected no error, got %v", err)
}
})
if !strings.Contains(output, `level=DEBUG msg="OPENAI_API_KEY detected for codex (masked: `) {
if !strings.Contains(output, `level=DEBUG msg="OPENAI_API_KEY detected for codex (source=agent_config)"`) {
t.Fatalf("expected api key log, got %q", output)
}
if !strings.Contains(output, credential.MaskAPIKey(apiKey)) {
t.Fatalf("expected masked key log, got %q", output)
if strings.Contains(output, apiKey) || strings.Contains(output, "****") {
t.Fatalf("key material leaked into log: %q", output)
}
if !strings.Contains(output, `level=DEBUG msg="OPENAI_BASE_URL detected for codex (value: https://openai.example.com/v1, source=agent_config)"`) {
t.Fatalf("expected base url log, got %q", output)
Expand Down
12 changes: 6 additions & 6 deletions internal/agent/custom.go
Original file line number Diff line number Diff line change
Expand Up @@ -290,15 +290,15 @@ func (a *CustomAgent) clearStaleOutputFile(ctx context.Context, rt Runtime, outp
// of the workspace.
safe, err := workspacePath(rt, outputFile)
if err != nil {
logging.WarnContextf(ctx, "CustomAgent: refusing to clear output file %s before run: %v", outputFile, err)
logging.WarnContextf(ctx, "CustomAgent: refusing to clear output file %s before run: %s", a.maskAPIKey(outputFile), a.maskAPIKey(err.Error()))
return false
}
outputFile = safe
q := shellQuote(outputFile)
cmd := "if [ -e " + q + " ]; then rm -f -- " + q + " && printf %s " + shellQuote(customStaleClearedMarker) + "; fi"
result, err := rt.Exec(ctx, cmd, ExecOptions{})
if err != nil {
logging.DebugContextf(ctx, "CustomAgent: could not clear stale output file %s: %v", outputFile, err)
logging.DebugContextf(ctx, "CustomAgent: could not clear stale output file %s: %s", a.maskAPIKey(outputFile), a.maskAPIKey(err.Error()))
return false
}
return strings.Contains(result.Stdout, customStaleClearedMarker)
Expand Down Expand Up @@ -466,7 +466,7 @@ func (a *CustomAgent) readRawResult(ctx context.Context, rt Runtime, custom *cus
if outputFile != "" {
safe, err := workspacePath(rt, outputFile)
if err != nil {
logging.WarnContextf(ctx, "CustomAgent: refusing to read output file %s after run: %v", outputFile, err)
logging.WarnContextf(ctx, "CustomAgent: refusing to read output file %s after run: %s", a.maskAPIKey(outputFile), a.maskAPIKey(err.Error()))
return strings.TrimSpace(result.Stdout), false
}
outputFile = safe
Expand All @@ -479,21 +479,21 @@ func (a *CustomAgent) readRawResult(ctx context.Context, rt Runtime, custom *cus
// same output_file basename.
tmpFile, err := os.CreateTemp("", "skill-up-custom-result-*")
if err != nil {
logging.WarnContextf(ctx, "CustomAgent: cannot create temp file for output_file %s, falling back to stdout: %v", outputFile, err)
logging.WarnContextf(ctx, "CustomAgent: cannot create temp file for output_file %s, falling back to stdout: %s", a.maskAPIKey(outputFile), a.maskAPIKey(err.Error()))
return result.Stdout, false
}
tmp := tmpFile.Name()
_ = tmpFile.Close()
defer func() { _ = os.Remove(tmp) }()

if err := rt.DownloadFile(ctx, outputFile, tmp); err != nil {
logging.WarnContextf(ctx, "CustomAgent: cannot read output_file %s, falling back to stdout: %v", outputFile, err)
logging.WarnContextf(ctx, "CustomAgent: cannot read output_file %s, falling back to stdout: %s", a.maskAPIKey(outputFile), a.maskAPIKey(err.Error()))
return result.Stdout, false
}
// The output file exists in the runtime; it is "produced" even if empty.
data, err := os.ReadFile(tmp)
if err != nil {
logging.WarnContextf(ctx, "CustomAgent: cannot read output_file %s, falling back to stdout: %v", outputFile, err)
logging.WarnContextf(ctx, "CustomAgent: cannot read output_file %s, falling back to stdout: %s", a.maskAPIKey(outputFile), a.maskAPIKey(err.Error()))
return result.Stdout, true
}
if strings.TrimSpace(string(data)) == "" {
Expand Down
36 changes: 36 additions & 0 deletions internal/agent/custom_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/alibaba/skill-up/internal/config"
"github.com/alibaba/skill-up/internal/credential"
"github.com/alibaba/skill-up/internal/logging"
"github.com/alibaba/skill-up/internal/runtime"
"github.com/alibaba/skill-up/pkg/transcript"
)
Expand All @@ -29,6 +30,41 @@ func customLocalAgent(custom *config.CustomEngineConfig) *CustomAgent {
return NewCustomAgent(Config{Name: "my-agent", Custom: custom})
}

func TestCustomAgent_OutputFileLogsRedactSecrets(t *testing.T) {
logging.SetVerbosity(1)
defer logging.SetVerbosity(0)

for _, source := range []string{"api_key", "custom_env"} {
t.Run(source, func(t *testing.T) {
rt := newCustomTestRuntime(t)
secret := strings.Join([]string{"output", "file", "secret"}, "-")
custom := &config.CustomEngineConfig{Local: &config.CustomLocalConfig{OutputFile: "configured"}}
ag := customLocalAgent(custom)
if source == "api_key" {
ag.Cfg.APIKey = secret
} else {
custom.Env = map[string]string{"CUSTOM_TOKEN": secret}
}
outside := filepath.Join(t.TempDir(), secret+".json")
inside := filepath.Join(rt.Workspace(), secret+".json")
output := captureStdout(t, func() {
if ag.clearStaleOutputFile(context.Background(), rt, outside) {
t.Fatal("cleared a path outside the workspace")
}
for _, path := range []string{outside, inside} {
raw, produced := ag.readRawResult(context.Background(), rt, custom, ExecResult{Stdout: "fallback"}, path)
if raw != "fallback" || produced {
t.Fatalf("readRawResult = (%q, %v), want stdout fallback", raw, produced)
}
}
})
if strings.Contains(output, secret) || !strings.Contains(output, "***REDACTED***") {
t.Fatalf("output file diagnostics did not redact secrets: %q", output)
}
})
}
}

func userMessages() []transcript.Message {
return []transcript.Message{{Role: transcript.RoleUser, Content: "review the diff"}}
}
Expand Down
4 changes: 2 additions & 2 deletions internal/credential/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ At a minimum, the following three categories should be distinguished.
Used to describe what the resolver / env scan found:

```text
[CREDENTIAL_DISCOVERED] provider=anthropic source=env api_key=sk****xx
[CREDENTIAL_DISCOVERED] provider=anthropic source=env api_key=present
[CREDENTIAL_DISCOVERED] provider=openai source=file base_url=https://...
```

Expand Down Expand Up @@ -332,7 +332,7 @@ These logs are particularly important for qodercli; otherwise users may mistaken
- When a CLI argument is accepted but ignored by the agent, explicitly note "the argument has been parsed but is not consumed by the current agent"
- Logs should include `kind=runner|judge`
- Logs should include `engine=<engine-name>`
- Sensitive values must be masked
- API keys must be logged as presence only, without key fragments

## Relation to the Current Implementation

Expand Down
15 changes: 2 additions & 13 deletions internal/credential/credential.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ const (
EnvQoderCNAccessToken = "QODER_CN_ACCESS_TOKEN" //nolint:gosec // local secret alias, not a credential value
)

const apiKeyMaskLen = 4 // Length for API key masking

// DefaultConfPath returns the default credentials file path,
// ~/.skill-up/credentials.yaml. It returns "" when the home directory
// cannot be determined, in which case the file layer is silently skipped.
Expand Down Expand Up @@ -92,16 +90,7 @@ func (r *Resolver) Get(provider string) (*config.APIKeyConfig, bool) {
return cred, ok
}

// MaskAPIKey masks the middle portion of an API key, showing first 2 and last 2 chars.
func MaskAPIKey(key string) string {
if len(key) <= apiKeyMaskLen {
return "****"
}

return key[:2] + "****" + key[len(key)-2:]
}

// logEffectiveConfig prints the currently loaded credentials (API_KEY masked).
// logEffectiveConfig prints credential presence without including key material.
func (r *Resolver) logEffectiveConfig() {
if len(r.creds) == 0 {
logging.Debugf("No credentials loaded from config file.")
Expand All @@ -110,7 +99,7 @@ func (r *Resolver) logEffectiveConfig() {
}
for name, cred := range r.creds {
if cred.APIKey != "" {
logging.Debugf("CREDENTIAL_DISCOVERED provider=%s source=resolver api_key=%s", name, MaskAPIKey(cred.APIKey))
logging.Debugf("CREDENTIAL_DISCOVERED provider=%s source=resolver api_key=present", name)
}
if cred.BaseURL != "" {
logging.Debugf("CREDENTIAL_DISCOVERED provider=%s source=resolver base_url=%s", name, cred.BaseURL)
Expand Down
42 changes: 16 additions & 26 deletions internal/credential/credential_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,32 +99,6 @@ func TestResolveRunnerConfig_OpenCodeDoesNotInferProviderFromModel(t *testing.T)
}
}

func TestMaskAPIKey(t *testing.T) {
t.Parallel()

tests := []struct {
name string
key string
expected string
}{
{name: "normal key", key: "sk-abcdefghijklmnopqrstuvwxyz123456", expected: "sk****56"},
{name: "short key (4 chars)", key: "abcd", expected: "****"},
{name: "shorter than mask length", key: "abc", expected: "****"},
{name: "empty key", key: "", expected: "****"},
{name: "exactly 5 chars", key: "abcde", expected: "ab****de"},
{name: "anthropic key format", key: "sk-ant-api03-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", expected: "sk****xx"},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := MaskAPIKey(tt.key); got != tt.expected {
t.Errorf("MaskAPIKey(%q) = %q, want %q", tt.key, got, tt.expected)
}
})
}
}

func TestDefaultConfPath(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -432,6 +406,22 @@ func TestResolveRunnerConfig_PreservesOpaqueSlashedCLIModel(t *testing.T) {
}
}

func TestResolver_LogEffectiveConfigDoesNotLogKeyMaterial(t *testing.T) {
logging.SetVerbosity(1)
defer logging.SetVerbosity(0)

apiKey := strings.Join([]string{"resolver", "test", "credential"}, "-")
r := NewResolver("")
r.creds["openai"] = &config.APIKeyConfig{APIKey: apiKey}
output := captureLogOutput(t, r.logEffectiveConfig)
if !strings.Contains(output, "api_key=present") {
t.Fatalf("expected credential-presence log, got %q", output)
}
if strings.Contains(output, apiKey) || strings.Contains(output, "****") {
t.Fatalf("key material leaked into log: %q", output)
}
}

func TestResolveRunnerConfig_DoesNotLogCLIAPIKey(t *testing.T) {
logging.SetVerbosity(1)
defer logging.SetVerbosity(0)
Expand Down
2 changes: 1 addition & 1 deletion internal/userconfig/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func MergeKwargs(c Config, envType string, evalKwargs map[string]string) map[str
return evalKwargs
}

out := make(map[string]string, len(userKwargs)+len(evalKwargs))
out := make(map[string]string, len(userKwargs))
maps.Copy(out, userKwargs)
maps.Copy(out, evalKwargs)
return out
Expand Down
18 changes: 18 additions & 0 deletions internal/userconfig/apply_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package userconfig

import (
"context"
"maps"
"os"
"slices"
"testing"
Expand Down Expand Up @@ -99,6 +100,23 @@ func TestMergeKwargs_NilEval(t *testing.T) {
}
}

func TestMergeKwargs_PreservesInputs(t *testing.T) {
t.Parallel()

userKwargs := Kwargs{"shared": "user", "user-only": "default"}
evalKwargs := map[string]string{"shared": "eval", "eval-only": "explicit"}
c := Config{RuntimeKwargs: map[string]Kwargs{"opensandbox": userKwargs}}
merged := MergeKwargs(c, "opensandbox", evalKwargs)
want := map[string]string{"shared": "eval", "user-only": "default", "eval-only": "explicit"}
if !maps.Equal(merged, want) {
t.Fatalf("merged = %v, want %v", merged, want)
}
merged["shared"] = "changed"
if userKwargs["shared"] != "user" || evalKwargs["shared"] != "eval" {
t.Fatal("merged map aliases an input")
}
}

func TestConfigContextRoundTrip(t *testing.T) {
t.Parallel()

Expand Down
Loading
Loading