eval/examples/basic_suite.json at main · kujolang/eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
{
  "name": "basic-eval-suite",
  "description": "Expected-fail intro suite: demonstrates core eval checks and includes one deliberate failure for report demos.",
  "version": "1.0.0",
  "output_dir": "./eval_results",
  "snapshot_dir": "./snapshots",
  "stop_on_failure": false,
  "tests": [
    {
      "name": "echo command succeeds",
      "description": "Verify a basic shell command exits cleanly.",
      "check": "command_succeeds",
      "params": {
        "command": "echo 'Hello from Eval'"
      }
    },
    {
      "name": "echo output contains greeting",
      "description": "Verify command output contains expected text.",
      "check": "output_contains",
      "params": {
        "command": "echo 'Hello from Eval'",
        "expected": "Hello"
      }
    },
    {
      "name": "echo output does not contain error",
      "description": "Verify command output does NOT contain error text.",
      "check": "output_does_not_contain",
      "params": {
        "command": "echo 'Everything is fine'",
        "expected": "ERROR"
      }
    },
    {
      "name": "eval.json file exists",
      "description": "Verify the eval config file exists on disk.",
      "check": "file_exists",
      "params": {
        "path": "eval.json"
      }
    },
    {
      "name": "temp file does not exist",
      "description": "Verify a non-existent file is reported as missing.",
      "check": "file_does_not_exist",
      "params": {
        "path": "./nonexistent_temp_file_xyz.txt"
      }
    },
    {
      "name": "kennel.toml contains package name",
      "description": "Verify a file contains expected text.",
      "check": "file_contains",
      "params": {
        "path": "kennel.toml",
        "expected": "eval"
      }
    },
    {
      "name": "kennel.toml does not contain forbidden text",
      "description": "Verify a file does NOT contain unexpected text.",
      "check": "file_does_not_contain",
      "params": {
        "path": "kennel.toml",
        "expected": "FORBIDDEN_TEXT_XYZ_123"
      }
    },
    {
      "name": "JSON shape validation passes",
      "description": "Verify a JSON file has expected keys.",
      "check": "json_matches_shape",
      "params": {
        "path": "kennel.toml",
        "required_keys": ["package"]
      }
    },
    {
      "name": "exit code is 0 for echo",
      "description": "Verify a command exits with a specific code.",
      "check": "exit_code_equals",
      "params": {
        "command": "echo ok",
        "expected": 0
      }
    },
    {
      "name": "expected-fail deliberate failure example",
      "description": "This test intentionally fails to show failure reporting.",
      "check": "command_fails",
      "params": {
        "command": "echo 'this succeeds, so the check fails'"
      }
    }
  ]
}