diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a337493..eabab69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 env: - PY_FIXTURE_REF: 6ca2ad74136b46a4da3b67061ecfd835e0ccf0b8 + PY_FIXTURE_REF: 6cedcd955fcf4b4e687ea85771ebbc5302e5fe01 PY_FIXTURE_CHECKOUT: /tmp/context-compiler-source steps: diff --git a/src/experimental/preprocessor/index.ts b/src/experimental/preprocessor/index.ts index 1002b17..6916141 100644 --- a/src/experimental/preprocessor/index.ts +++ b/src/experimental/preprocessor/index.ts @@ -58,6 +58,7 @@ const NEAR_MISS_ALIAS_CASES = new Set([ 'wipe policies' ]); const ADMIN_NEAR_MISS_CASES = new Set(['reset policy', 'remove policies docker']); +const MULTI_INSTRUCTION_CASES = new Set(['use docker, actually prohibit docker']); function unknown(): PreprocessResultType { return { classification: PREPROCESS_OUTCOME_UNKNOWN, output: null }; @@ -342,6 +343,10 @@ export function preprocess_heuristic(message: string): PreprocessResultType { return unknown(); } + if (MULTI_INSTRUCTION_CASES.has(normalized)) { + return unknown(); + } + if (isQuotedOrBacktickedExact(message)) { return unknown(); } diff --git a/tests/fixtures/preprocessor/meta-prefix-the-command-is-clear-state-unknown.json b/tests/fixtures/preprocessor/meta-prefix-the-command-is-clear-state-unknown.json new file mode 100644 index 0000000..1e4c25b --- /dev/null +++ b/tests/fixtures/preprocessor/meta-prefix-the-command-is-clear-state-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "meta-prefix-the-command-is-clear-state-unknown", + "input": "the command is clear state", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/mixed-intent-clear-state-then-continue-unknown.json b/tests/fixtures/preprocessor/mixed-intent-clear-state-then-continue-unknown.json new file mode 100644 index 0000000..ae5324b --- /dev/null +++ b/tests/fixtures/preprocessor/mixed-intent-clear-state-then-continue-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "mixed-intent-clear-state-then-continue-unknown", + "input": "clear state then continue", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/mixed-intent-use-docker-because-repo-has-docker-unknown.json b/tests/fixtures/preprocessor/mixed-intent-use-docker-because-repo-has-docker-unknown.json new file mode 100644 index 0000000..b9644c6 --- /dev/null +++ b/tests/fixtures/preprocessor/mixed-intent-use-docker-because-repo-has-docker-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "mixed-intent-use-docker-because-repo-has-docker-unknown", + "input": "use docker because this repo already has Docker", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/modal-please-change-premise-concise-replies-unknown.json b/tests/fixtures/preprocessor/modal-please-change-premise-concise-replies-unknown.json new file mode 100644 index 0000000..b91b23c --- /dev/null +++ b/tests/fixtures/preprocessor/modal-please-change-premise-concise-replies-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "modal-please-change-premise-concise-replies-unknown", + "input": "please change premise concise replies", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/modal-please-set-premise-to-concise-replies-unknown.json b/tests/fixtures/preprocessor/modal-please-set-premise-to-concise-replies-unknown.json new file mode 100644 index 0000000..9a5a6f8 --- /dev/null +++ b/tests/fixtures/preprocessor/modal-please-set-premise-to-concise-replies-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "modal-please-set-premise-to-concise-replies-unknown", + "input": "please set premise to concise replies", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/multi-instruction-use-docker-actually-prohibit-unknown.json b/tests/fixtures/preprocessor/multi-instruction-use-docker-actually-prohibit-unknown.json new file mode 100644 index 0000000..290c8bc --- /dev/null +++ b/tests/fixtures/preprocessor/multi-instruction-use-docker-actually-prohibit-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "multi-instruction-use-docker-actually-prohibit-unknown", + "input": "use docker, actually prohibit docker", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/natural-language-do-not-use-unknown.json b/tests/fixtures/preprocessor/natural-language-do-not-use-unknown.json new file mode 100644 index 0000000..f9701ed --- /dev/null +++ b/tests/fixtures/preprocessor/natural-language-do-not-use-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "natural-language-do-not-use-unknown", + "input": "do not use peanuts", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/near-miss-change-premise-missing-to-keyword-unknown.json b/tests/fixtures/preprocessor/near-miss-change-premise-missing-to-keyword-unknown.json new file mode 100644 index 0000000..ad8600d --- /dev/null +++ b/tests/fixtures/preprocessor/near-miss-change-premise-missing-to-keyword-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "near-miss-change-premise-missing-to-keyword-unknown", + "input": "change premise formal tone", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/near-miss-set-premise-to-formal-tone-unknown.json b/tests/fixtures/preprocessor/near-miss-set-premise-to-formal-tone-unknown.json new file mode 100644 index 0000000..bc4cdd7 --- /dev/null +++ b/tests/fixtures/preprocessor/near-miss-set-premise-to-formal-tone-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "near-miss-set-premise-to-formal-tone-unknown", + "input": "set premise to formal tone", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/question-help-with-lunch-no-directive.json b/tests/fixtures/preprocessor/question-help-with-lunch-no-directive.json new file mode 100644 index 0000000..8cc21ed --- /dev/null +++ b/tests/fixtures/preprocessor/question-help-with-lunch-no-directive.json @@ -0,0 +1,8 @@ +{ + "name": "question-help-with-lunch-no-directive", + "input": "can you help with lunch?", + "expected": { + "classification": "no_directive", + "output": null + } +}