From 5a4629f8701e4c24e51a4466109825911e5eba09 Mon Sep 17 00:00:00 2001 From: drewOrc <36374426+drewOrc@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:25:39 +0800 Subject: [PATCH] results: majority and TF-IDF baselines, 36 of 36 points Two baselines x six k x three seeds on the same curve_sample(k, seed) as the encoders. Produced at 17:22 before the completion checks existed and re-verified with them: the index rebuilt from disk holds 36 unique points and every archive matches the manifest SHA-256 (archives were not recomputed). Logits archives go to a release. --- results/curves/baselines.json | 420 +++++++++++++++++++ results/logits-manifest.json | 396 +++++++++++++++++ results/runs/majority-k1-seed42.json | 82 ++++ results/runs/majority-k1-seed43.json | 82 ++++ results/runs/majority-k1-seed44.json | 82 ++++ results/runs/majority-k10-seed42.json | 82 ++++ results/runs/majority-k10-seed43.json | 82 ++++ results/runs/majority-k10-seed44.json | 82 ++++ results/runs/majority-k100-seed42.json | 82 ++++ results/runs/majority-k100-seed43.json | 82 ++++ results/runs/majority-k100-seed44.json | 82 ++++ results/runs/majority-k25-seed42.json | 82 ++++ results/runs/majority-k25-seed43.json | 82 ++++ results/runs/majority-k25-seed44.json | 82 ++++ results/runs/majority-k5-seed42.json | 82 ++++ results/runs/majority-k5-seed43.json | 82 ++++ results/runs/majority-k5-seed44.json | 82 ++++ results/runs/majority-k50-seed42.json | 82 ++++ results/runs/majority-k50-seed43.json | 82 ++++ results/runs/majority-k50-seed44.json | 82 ++++ results/runs/tfidf-centroid-k1-seed42.json | 88 ++++ results/runs/tfidf-centroid-k1-seed43.json | 88 ++++ results/runs/tfidf-centroid-k1-seed44.json | 88 ++++ results/runs/tfidf-centroid-k10-seed42.json | 88 ++++ results/runs/tfidf-centroid-k10-seed43.json | 88 ++++ results/runs/tfidf-centroid-k10-seed44.json | 88 ++++ results/runs/tfidf-centroid-k100-seed42.json | 88 ++++ results/runs/tfidf-centroid-k100-seed43.json | 88 ++++ results/runs/tfidf-centroid-k100-seed44.json | 88 ++++ results/runs/tfidf-centroid-k25-seed42.json | 88 ++++ results/runs/tfidf-centroid-k25-seed43.json | 88 ++++ results/runs/tfidf-centroid-k25-seed44.json | 88 ++++ results/runs/tfidf-centroid-k5-seed42.json | 88 ++++ results/runs/tfidf-centroid-k5-seed43.json | 88 ++++ results/runs/tfidf-centroid-k5-seed44.json | 88 ++++ results/runs/tfidf-centroid-k50-seed42.json | 88 ++++ results/runs/tfidf-centroid-k50-seed43.json | 88 ++++ results/runs/tfidf-centroid-k50-seed44.json | 88 ++++ 38 files changed, 3876 insertions(+) create mode 100644 results/curves/baselines.json create mode 100644 results/runs/majority-k1-seed42.json create mode 100644 results/runs/majority-k1-seed43.json create mode 100644 results/runs/majority-k1-seed44.json create mode 100644 results/runs/majority-k10-seed42.json create mode 100644 results/runs/majority-k10-seed43.json create mode 100644 results/runs/majority-k10-seed44.json create mode 100644 results/runs/majority-k100-seed42.json create mode 100644 results/runs/majority-k100-seed43.json create mode 100644 results/runs/majority-k100-seed44.json create mode 100644 results/runs/majority-k25-seed42.json create mode 100644 results/runs/majority-k25-seed43.json create mode 100644 results/runs/majority-k25-seed44.json create mode 100644 results/runs/majority-k5-seed42.json create mode 100644 results/runs/majority-k5-seed43.json create mode 100644 results/runs/majority-k5-seed44.json create mode 100644 results/runs/majority-k50-seed42.json create mode 100644 results/runs/majority-k50-seed43.json create mode 100644 results/runs/majority-k50-seed44.json create mode 100644 results/runs/tfidf-centroid-k1-seed42.json create mode 100644 results/runs/tfidf-centroid-k1-seed43.json create mode 100644 results/runs/tfidf-centroid-k1-seed44.json create mode 100644 results/runs/tfidf-centroid-k10-seed42.json create mode 100644 results/runs/tfidf-centroid-k10-seed43.json create mode 100644 results/runs/tfidf-centroid-k10-seed44.json create mode 100644 results/runs/tfidf-centroid-k100-seed42.json create mode 100644 results/runs/tfidf-centroid-k100-seed43.json create mode 100644 results/runs/tfidf-centroid-k100-seed44.json create mode 100644 results/runs/tfidf-centroid-k25-seed42.json create mode 100644 results/runs/tfidf-centroid-k25-seed43.json create mode 100644 results/runs/tfidf-centroid-k25-seed44.json create mode 100644 results/runs/tfidf-centroid-k5-seed42.json create mode 100644 results/runs/tfidf-centroid-k5-seed43.json create mode 100644 results/runs/tfidf-centroid-k5-seed44.json create mode 100644 results/runs/tfidf-centroid-k50-seed42.json create mode 100644 results/runs/tfidf-centroid-k50-seed43.json create mode 100644 results/runs/tfidf-centroid-k50-seed44.json diff --git a/results/curves/baselines.json b/results/curves/baselines.json new file mode 100644 index 0000000..4e76468 --- /dev/null +++ b/results/curves/baselines.json @@ -0,0 +1,420 @@ +{ + "baselines": { + "majority": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "tfidf-centroid": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + } + }, + "points": [ + { + "baseline": "majority", + "k": 1, + "seed": 42, + "run_name": "majority-k1-seed42", + "logits_file": "majority-k1-seed42.npz", + "logits_sha256": "ad9e1099ed7d88354021da1cda7ae310d1313af9d7d4654eafd03fabb8318e34", + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "8f04aca1ffa32368a0e811795f70067f9d2b6d1551e10a3be5e08f5c954db166" + }, + { + "baseline": "tfidf-centroid", + "k": 1, + "seed": 42, + "run_name": "tfidf-centroid-k1-seed42", + "logits_file": "tfidf-centroid-k1-seed42.npz", + "logits_sha256": "94258de1803c08b0ba3fd2066c15b4814dfb701f499959b33319623e70d6089d", + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "8f04aca1ffa32368a0e811795f70067f9d2b6d1551e10a3be5e08f5c954db166" + }, + { + "baseline": "majority", + "k": 1, + "seed": 43, + "run_name": "majority-k1-seed43", + "logits_file": "majority-k1-seed43.npz", + "logits_sha256": "1b6de003a49322dea12ae7a224e9e6934b084fbbf6014bf7cb6721950b9d80e6", + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "4c659defaf9b9c6f28f4d5e67ef92b466627cdb200623552ba46ced1d2750c9c" + }, + { + "baseline": "tfidf-centroid", + "k": 1, + "seed": 43, + "run_name": "tfidf-centroid-k1-seed43", + "logits_file": "tfidf-centroid-k1-seed43.npz", + "logits_sha256": "d2d86e9f3d9cb80ca61f3323aa17dcb00ce80afbddf52beddf89da9ccb841380", + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "4c659defaf9b9c6f28f4d5e67ef92b466627cdb200623552ba46ced1d2750c9c" + }, + { + "baseline": "majority", + "k": 1, + "seed": 44, + "run_name": "majority-k1-seed44", + "logits_file": "majority-k1-seed44.npz", + "logits_sha256": "52a40757315fd9be80c2fd0cf5329f0b4c446ff25cf9a23ff714beab956155c9", + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "d69635e70893d70759a46fd0fa54561eae262653eca95898076348b9c660b96a" + }, + { + "baseline": "tfidf-centroid", + "k": 1, + "seed": 44, + "run_name": "tfidf-centroid-k1-seed44", + "logits_file": "tfidf-centroid-k1-seed44.npz", + "logits_sha256": "fe1f9f4b8c84c652cec9717dd20fd94591d7134748764883360c9fe27ef5dbc5", + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "d69635e70893d70759a46fd0fa54561eae262653eca95898076348b9c660b96a" + }, + { + "baseline": "majority", + "k": 5, + "seed": 42, + "run_name": "majority-k5-seed42", + "logits_file": "majority-k5-seed42.npz", + "logits_sha256": "9e6bb6f163ed274f4c53aedd58604b2c1259d3c083384ea7fb2d49455f9db466", + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "bfabf08971d3a16a502e25b2d5a7f1ce204504a526d5b3cb34fec9346dd8caa7" + }, + { + "baseline": "tfidf-centroid", + "k": 5, + "seed": 42, + "run_name": "tfidf-centroid-k5-seed42", + "logits_file": "tfidf-centroid-k5-seed42.npz", + "logits_sha256": "cac96d18d9cad091f8dd3822554c9b94ae38e08d90688d161f52560791ccb63b", + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "bfabf08971d3a16a502e25b2d5a7f1ce204504a526d5b3cb34fec9346dd8caa7" + }, + { + "baseline": "majority", + "k": 5, + "seed": 43, + "run_name": "majority-k5-seed43", + "logits_file": "majority-k5-seed43.npz", + "logits_sha256": "3314ad9a70b585c6f4fccb9eb0bd56a5df209206e20b552c464727e10bbaba98", + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "4e289193d9669c26c40d083246f6c34426bca5ff041ac526796a5f976ca9c638" + }, + { + "baseline": "tfidf-centroid", + "k": 5, + "seed": 43, + "run_name": "tfidf-centroid-k5-seed43", + "logits_file": "tfidf-centroid-k5-seed43.npz", + "logits_sha256": "dbfdd8f7f5107d61cae3e48f2e7a99916ecdf6a094047b4053e65fe43bc3ec1d", + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "4e289193d9669c26c40d083246f6c34426bca5ff041ac526796a5f976ca9c638" + }, + { + "baseline": "majority", + "k": 5, + "seed": 44, + "run_name": "majority-k5-seed44", + "logits_file": "majority-k5-seed44.npz", + "logits_sha256": "1da11415daed0f054fcb2477402f323bdcc4c0908abfebacfc116bc6f1ec5837", + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "ba57c52c7c416c611ba215bb10a393fba470ec511aaebad2b416914ca674988d" + }, + { + "baseline": "tfidf-centroid", + "k": 5, + "seed": 44, + "run_name": "tfidf-centroid-k5-seed44", + "logits_file": "tfidf-centroid-k5-seed44.npz", + "logits_sha256": "e3e751ba605ba46a492f6f8ba0759c63b41239d46bf01f9513dc0d1d5db175ba", + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "ba57c52c7c416c611ba215bb10a393fba470ec511aaebad2b416914ca674988d" + }, + { + "baseline": "majority", + "k": 10, + "seed": 42, + "run_name": "majority-k10-seed42", + "logits_file": "majority-k10-seed42.npz", + "logits_sha256": "73c39d0e3e96319a1a3e339c4f7beeb57fa9c05ffbdda933ad50d8da0b47ce42", + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "2606cc878d85759a5885e1b9be35205023b156e6cf8a4eea85bd635c3b492e74" + }, + { + "baseline": "tfidf-centroid", + "k": 10, + "seed": 42, + "run_name": "tfidf-centroid-k10-seed42", + "logits_file": "tfidf-centroid-k10-seed42.npz", + "logits_sha256": "efa61924b5913a37d54b9d4a3d13cd5be52fa35b56491dbfa9709fc8755b1aef", + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "2606cc878d85759a5885e1b9be35205023b156e6cf8a4eea85bd635c3b492e74" + }, + { + "baseline": "majority", + "k": 10, + "seed": 43, + "run_name": "majority-k10-seed43", + "logits_file": "majority-k10-seed43.npz", + "logits_sha256": "10fd16db95cd9c12358fdb5a5d5d17c0ca0f6248c6f2c7506e03c8d30da5d650", + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "072190bdfc028a2e0bc6ec85d39e26ae1b27854281c6226be57b8f8b9d20e165" + }, + { + "baseline": "tfidf-centroid", + "k": 10, + "seed": 43, + "run_name": "tfidf-centroid-k10-seed43", + "logits_file": "tfidf-centroid-k10-seed43.npz", + "logits_sha256": "aa4fc1ec5e6cdcac08cd404d778130f36725e6151a54f9ab2be795d38ed523e2", + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "072190bdfc028a2e0bc6ec85d39e26ae1b27854281c6226be57b8f8b9d20e165" + }, + { + "baseline": "majority", + "k": 10, + "seed": 44, + "run_name": "majority-k10-seed44", + "logits_file": "majority-k10-seed44.npz", + "logits_sha256": "68305d812ce5fa3dc38d311014e4231cb19cb2a5fc25548011c3d448accfd491", + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "31a30630bb8f711a55173e8837eaf4500c2c5dcd5efe461b561a04e843cddd55" + }, + { + "baseline": "tfidf-centroid", + "k": 10, + "seed": 44, + "run_name": "tfidf-centroid-k10-seed44", + "logits_file": "tfidf-centroid-k10-seed44.npz", + "logits_sha256": "cfb9163965d84af33aef86d28145073b143ebb604bd9901d37b18d54d842db22", + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "31a30630bb8f711a55173e8837eaf4500c2c5dcd5efe461b561a04e843cddd55" + }, + { + "baseline": "majority", + "k": 25, + "seed": 42, + "run_name": "majority-k25-seed42", + "logits_file": "majority-k25-seed42.npz", + "logits_sha256": "0847cf359ec51cb09bfec7ae8dfea24c09254963a5ea73a82875b4d1780c593e", + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "51184b580bdbe58ab700d2545887a30c442950389b98497c3b7edaa7ca47174c" + }, + { + "baseline": "tfidf-centroid", + "k": 25, + "seed": 42, + "run_name": "tfidf-centroid-k25-seed42", + "logits_file": "tfidf-centroid-k25-seed42.npz", + "logits_sha256": "be4fdf7bd9f789ab99de9042ffbc8b0d41d6cb152e08a7c079fd3bf623909a47", + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "51184b580bdbe58ab700d2545887a30c442950389b98497c3b7edaa7ca47174c" + }, + { + "baseline": "majority", + "k": 25, + "seed": 43, + "run_name": "majority-k25-seed43", + "logits_file": "majority-k25-seed43.npz", + "logits_sha256": "1d6145a48b53b954eba6a1d453b77f5eafc5f9cada582ff9012f7d39f1201ad5", + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "92f91be6e0948dbc01e82b81485c72829dd232443955fa0ae214b1451a0a704f" + }, + { + "baseline": "tfidf-centroid", + "k": 25, + "seed": 43, + "run_name": "tfidf-centroid-k25-seed43", + "logits_file": "tfidf-centroid-k25-seed43.npz", + "logits_sha256": "c09e6879bf292df29c7fa24208687bdc43a2513f62033313cc100d5d6a1f0fa4", + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "92f91be6e0948dbc01e82b81485c72829dd232443955fa0ae214b1451a0a704f" + }, + { + "baseline": "majority", + "k": 25, + "seed": 44, + "run_name": "majority-k25-seed44", + "logits_file": "majority-k25-seed44.npz", + "logits_sha256": "be5db7a3d7fda2f50097788b959e62a40acd27100a64ad4350f1527883183e1f", + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "790cd7cf9eb24143276f6c59c7bac29c4a0b42f14bef38e9634e96253e6315fb" + }, + { + "baseline": "tfidf-centroid", + "k": 25, + "seed": 44, + "run_name": "tfidf-centroid-k25-seed44", + "logits_file": "tfidf-centroid-k25-seed44.npz", + "logits_sha256": "e7aa6e711a2a6b5322a78def95a33738c6bf0f80f6006710d203af2c9c3fdcd5", + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "790cd7cf9eb24143276f6c59c7bac29c4a0b42f14bef38e9634e96253e6315fb" + }, + { + "baseline": "majority", + "k": 50, + "seed": 42, + "run_name": "majority-k50-seed42", + "logits_file": "majority-k50-seed42.npz", + "logits_sha256": "7c4872ed6ac1c993a1850633e26c44fac500db0efdc4923ea6a1ad1486a5ef97", + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "8a9a52f8ae41506d9847bd9ceba987edadc6bc3304487be22e664a95e1b9b18f" + }, + { + "baseline": "tfidf-centroid", + "k": 50, + "seed": 42, + "run_name": "tfidf-centroid-k50-seed42", + "logits_file": "tfidf-centroid-k50-seed42.npz", + "logits_sha256": "27608c5f570bcb13a37d9cff167a0c2980f2915bc0fbce190c0027b0e52c3b93", + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "8a9a52f8ae41506d9847bd9ceba987edadc6bc3304487be22e664a95e1b9b18f" + }, + { + "baseline": "majority", + "k": 50, + "seed": 43, + "run_name": "majority-k50-seed43", + "logits_file": "majority-k50-seed43.npz", + "logits_sha256": "ddd5754bd4acf617e268c6c68c49b3ab257bef78fb1e41e56cdb2d5508c3638e", + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "836ad6186d05a042ae00b3ab45efc560eb45f774644bf5e4f9b13a6e66d82982" + }, + { + "baseline": "tfidf-centroid", + "k": 50, + "seed": 43, + "run_name": "tfidf-centroid-k50-seed43", + "logits_file": "tfidf-centroid-k50-seed43.npz", + "logits_sha256": "9f8eed2728260b4b58e05b7e7d70b79b4b5f74d6b9f7a01decc8cff13e42aa2f", + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "836ad6186d05a042ae00b3ab45efc560eb45f774644bf5e4f9b13a6e66d82982" + }, + { + "baseline": "majority", + "k": 50, + "seed": 44, + "run_name": "majority-k50-seed44", + "logits_file": "majority-k50-seed44.npz", + "logits_sha256": "15276ebf3a368cb46ccb4beb03a529067369266cbfcfe1d5e6294e022d374e7c", + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "5db5ce1ae1864ba63a4164a799b09a7c1d783a4acbd55efc587613d986752013" + }, + { + "baseline": "tfidf-centroid", + "k": 50, + "seed": 44, + "run_name": "tfidf-centroid-k50-seed44", + "logits_file": "tfidf-centroid-k50-seed44.npz", + "logits_sha256": "3fa2c12a4a881fe0c9148c86acc9be6edd315d968621aecf6f7936c0927d79ab", + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "5db5ce1ae1864ba63a4164a799b09a7c1d783a4acbd55efc587613d986752013" + }, + { + "baseline": "majority", + "k": 100, + "seed": 42, + "run_name": "majority-k100-seed42", + "logits_file": "majority-k100-seed42.npz", + "logits_sha256": "45de4fe9a649263e6470be6fdbe46e8a4fceb5a97a0afe36c21062b7d6447fb4", + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + { + "baseline": "tfidf-centroid", + "k": 100, + "seed": 42, + "run_name": "tfidf-centroid-k100-seed42", + "logits_file": "tfidf-centroid-k100-seed42.npz", + "logits_sha256": "0ae4c761890317d76efc8d3b08a3920eab94633e32f710f0059cb7b96f82cb75", + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + { + "baseline": "majority", + "k": 100, + "seed": 43, + "run_name": "majority-k100-seed43", + "logits_file": "majority-k100-seed43.npz", + "logits_sha256": "d57231c421661de1827d36c2b11d1a9cba0d5107f57447d639b25a70bcc46bb6", + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + { + "baseline": "tfidf-centroid", + "k": 100, + "seed": 43, + "run_name": "tfidf-centroid-k100-seed43", + "logits_file": "tfidf-centroid-k100-seed43.npz", + "logits_sha256": "01e56ce77413fd0e6dcb11398fdd0eb8024997983daf7bac5fea2a410d1bbe33", + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + { + "baseline": "majority", + "k": 100, + "seed": 44, + "run_name": "majority-k100-seed44", + "logits_file": "majority-k100-seed44.npz", + "logits_sha256": "7422da060f87bd4325a17ef3f38948acf314349861f27c352038cbad2834c72b", + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + { + "baseline": "tfidf-centroid", + "k": 100, + "seed": 44, + "run_name": "tfidf-centroid-k100-seed44", + "logits_file": "tfidf-centroid-k100-seed44.npz", + "logits_sha256": "ae69c656b6f00892ace610769c997b03dcfe588cb8568b4cd4e94bba9fee14bf", + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + } + ] +} diff --git a/results/logits-manifest.json b/results/logits-manifest.json index b96459e..c73aa18 100644 --- a/results/logits-manifest.json +++ b/results/logits-manifest.json @@ -33,6 +33,402 @@ "validation": 3100, "test": 5500 } + }, + "majority-k1-seed42.npz": { + "sha256": "ad9e1099ed7d88354021da1cda7ae310d1313af9d7d4654eafd03fabb8318e34", + "bytes": 24098, + "run_name": "majority-k1-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:05+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k1-seed43.npz": { + "sha256": "1b6de003a49322dea12ae7a224e9e6934b084fbbf6014bf7cb6721950b9d80e6", + "bytes": 24098, + "run_name": "majority-k1-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:06+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k1-seed44.npz": { + "sha256": "52a40757315fd9be80c2fd0cf5329f0b4c446ff25cf9a23ff714beab956155c9", + "bytes": 24098, + "run_name": "majority-k1-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:06+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k10-seed42.npz": { + "sha256": "73c39d0e3e96319a1a3e339c4f7beeb57fa9c05ffbdda933ad50d8da0b47ce42", + "bytes": 24108, + "run_name": "majority-k10-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:08+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k10-seed43.npz": { + "sha256": "10fd16db95cd9c12358fdb5a5d5d17c0ca0f6248c6f2c7506e03c8d30da5d650", + "bytes": 24108, + "run_name": "majority-k10-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:08+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k10-seed44.npz": { + "sha256": "68305d812ce5fa3dc38d311014e4231cb19cb2a5fc25548011c3d448accfd491", + "bytes": 24107, + "run_name": "majority-k10-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:09+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k100-seed42.npz": { + "sha256": "45de4fe9a649263e6470be6fdbe46e8a4fceb5a97a0afe36c21062b7d6447fb4", + "bytes": 24111, + "run_name": "majority-k100-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:12+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k100-seed43.npz": { + "sha256": "d57231c421661de1827d36c2b11d1a9cba0d5107f57447d639b25a70bcc46bb6", + "bytes": 24111, + "run_name": "majority-k100-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:13+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k100-seed44.npz": { + "sha256": "7422da060f87bd4325a17ef3f38948acf314349861f27c352038cbad2834c72b", + "bytes": 24111, + "run_name": "majority-k100-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:13+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k25-seed42.npz": { + "sha256": "0847cf359ec51cb09bfec7ae8dfea24c09254963a5ea73a82875b4d1780c593e", + "bytes": 24108, + "run_name": "majority-k25-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:09+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k25-seed43.npz": { + "sha256": "1d6145a48b53b954eba6a1d453b77f5eafc5f9cada582ff9012f7d39f1201ad5", + "bytes": 24108, + "run_name": "majority-k25-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:10+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k25-seed44.npz": { + "sha256": "be5db7a3d7fda2f50097788b959e62a40acd27100a64ad4350f1527883183e1f", + "bytes": 24108, + "run_name": "majority-k25-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:10+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k5-seed42.npz": { + "sha256": "9e6bb6f163ed274f4c53aedd58604b2c1259d3c083384ea7fb2d49455f9db466", + "bytes": 24098, + "run_name": "majority-k5-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:07+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k5-seed43.npz": { + "sha256": "3314ad9a70b585c6f4fccb9eb0bd56a5df209206e20b552c464727e10bbaba98", + "bytes": 24097, + "run_name": "majority-k5-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:07+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k5-seed44.npz": { + "sha256": "1da11415daed0f054fcb2477402f323bdcc4c0908abfebacfc116bc6f1ec5837", + "bytes": 24098, + "run_name": "majority-k5-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:07+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k50-seed42.npz": { + "sha256": "7c4872ed6ac1c993a1850633e26c44fac500db0efdc4923ea6a1ad1486a5ef97", + "bytes": 24110, + "run_name": "majority-k50-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:11+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k50-seed43.npz": { + "sha256": "ddd5754bd4acf617e268c6c68c49b3ab257bef78fb1e41e56cdb2d5508c3638e", + "bytes": 24110, + "run_name": "majority-k50-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:11+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "majority-k50-seed44.npz": { + "sha256": "15276ebf3a368cb46ccb4beb03a529067369266cbfcfe1d5e6294e022d374e7c", + "bytes": 24110, + "run_name": "majority-k50-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:12+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k1-seed42.npz": { + "sha256": "94258de1803c08b0ba3fd2066c15b4814dfb701f499959b33319623e70d6089d", + "bytes": 2211462, + "run_name": "tfidf-centroid-k1-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:05+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k1-seed43.npz": { + "sha256": "d2d86e9f3d9cb80ca61f3323aa17dcb00ce80afbddf52beddf89da9ccb841380", + "bytes": 2387115, + "run_name": "tfidf-centroid-k1-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:06+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k1-seed44.npz": { + "sha256": "fe1f9f4b8c84c652cec9717dd20fd94591d7134748764883360c9fe27ef5dbc5", + "bytes": 2290727, + "run_name": "tfidf-centroid-k1-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:06+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k10-seed42.npz": { + "sha256": "efa61924b5913a37d54b9d4a3d13cd5be52fa35b56491dbfa9709fc8755b1aef", + "bytes": 4087246, + "run_name": "tfidf-centroid-k10-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:08+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k10-seed43.npz": { + "sha256": "aa4fc1ec5e6cdcac08cd404d778130f36725e6151a54f9ab2be795d38ed523e2", + "bytes": 4068984, + "run_name": "tfidf-centroid-k10-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:09+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k10-seed44.npz": { + "sha256": "cfb9163965d84af33aef86d28145073b143ebb604bd9901d37b18d54d842db22", + "bytes": 4080654, + "run_name": "tfidf-centroid-k10-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:09+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k100-seed42.npz": { + "sha256": "0ae4c761890317d76efc8d3b08a3920eab94633e32f710f0059cb7b96f82cb75", + "bytes": 4522480, + "run_name": "tfidf-centroid-k100-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:13+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k100-seed43.npz": { + "sha256": "01e56ce77413fd0e6dcb11398fdd0eb8024997983daf7bac5fea2a410d1bbe33", + "bytes": 4522481, + "run_name": "tfidf-centroid-k100-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:13+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k100-seed44.npz": { + "sha256": "ae69c656b6f00892ace610769c997b03dcfe588cb8568b4cd4e94bba9fee14bf", + "bytes": 4522482, + "run_name": "tfidf-centroid-k100-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:14+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k25-seed42.npz": { + "sha256": "be4fdf7bd9f789ab99de9042ffbc8b0d41d6cb152e08a7c079fd3bf623909a47", + "bytes": 4354875, + "run_name": "tfidf-centroid-k25-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:09+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k25-seed43.npz": { + "sha256": "c09e6879bf292df29c7fa24208687bdc43a2513f62033313cc100d5d6a1f0fa4", + "bytes": 4358905, + "run_name": "tfidf-centroid-k25-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:10+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k25-seed44.npz": { + "sha256": "e7aa6e711a2a6b5322a78def95a33738c6bf0f80f6006710d203af2c9c3fdcd5", + "bytes": 4349783, + "run_name": "tfidf-centroid-k25-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:10+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k5-seed42.npz": { + "sha256": "cac96d18d9cad091f8dd3822554c9b94ae38e08d90688d161f52560791ccb63b", + "bytes": 3697662, + "run_name": "tfidf-centroid-k5-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:07+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k5-seed43.npz": { + "sha256": "dbfdd8f7f5107d61cae3e48f2e7a99916ecdf6a094047b4053e65fe43bc3ec1d", + "bytes": 3708540, + "run_name": "tfidf-centroid-k5-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:07+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k5-seed44.npz": { + "sha256": "e3e751ba605ba46a492f6f8ba0759c63b41239d46bf01f9513dc0d1d5db175ba", + "bytes": 3769932, + "run_name": "tfidf-centroid-k5-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:08+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k50-seed42.npz": { + "sha256": "27608c5f570bcb13a37d9cff167a0c2980f2915bc0fbce190c0027b0e52c3b93", + "bytes": 4457536, + "run_name": "tfidf-centroid-k50-seed42", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:11+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k50-seed43.npz": { + "sha256": "9f8eed2728260b4b58e05b7e7d70b79b4b5f74d6b9f7a01decc8cff13e42aa2f", + "bytes": 4461339, + "run_name": "tfidf-centroid-k50-seed43", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:11+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } + }, + "tfidf-centroid-k50-seed44.npz": { + "sha256": "3fa2c12a4a881fe0c9148c86acc9be6edd315d968621aecf6f7936c0927d79ab", + "bytes": 4455620, + "run_name": "tfidf-centroid-k50-seed44", + "git_commit": "9eec09f16a9cf059c38884f3b8ede49957bd7e87", + "created_at": "2026-09-23T09:22:12+00:00", + "rows": { + "validation": 3100, + "test": 5500 + } } } } diff --git a/results/runs/majority-k1-seed42.json b/results/runs/majority-k1-seed42.json new file mode 100644 index 0000000..7c11336 --- /dev/null +++ b/results/runs/majority-k1-seed42.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k1-seed42", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 1, + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "8f04aca1ffa32368a0e811795f70067f9d2b6d1551e10a3be5e08f5c954db166" + }, + "logits": { + "file": "majority-k1-seed42.npz", + "sha256": "ad9e1099ed7d88354021da1cda7ae310d1313af9d7d4654eafd03fabb8318e34", + "bytes": 24098 + }, + "metrics": { + "temperature": 0.6826061908058604, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.012650220997595999, + "ece_8_summed": 0.003204722652724451, + "nll_151": 4.994998815055462, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 1.1779402661615812e-09, + "ece_8_summed": 2.9841140669617516e-10, + "nll_151": 4.991507764905244, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1622103382996488, + "ece_8_summed": 0.04109328570257781, + "nll_151": 4.830690229326394, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011612411252, + "ece_8_summed": 0.03788856275144198, + "nll_151": 4.750800036287824, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k1-seed43.json b/results/runs/majority-k1-seed43.json new file mode 100644 index 0000000..670eb08 --- /dev/null +++ b/results/runs/majority-k1-seed43.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k1-seed43", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 1, + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "4c659defaf9b9c6f28f4d5e67ef92b466627cdb200623552ba46ced1d2750c9c" + }, + "logits": { + "file": "majority-k1-seed43.npz", + "sha256": "1b6de003a49322dea12ae7a224e9e6934b084fbbf6014bf7cb6721950b9d80e6", + "bytes": 24098 + }, + "metrics": { + "temperature": 0.6826061908058604, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.012650220997595999, + "ece_8_summed": 0.003204722652724451, + "nll_151": 4.994998815055462, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 1.1779402661615812e-09, + "ece_8_summed": 2.9841140669617516e-10, + "nll_151": 4.991507764905244, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1622103382996488, + "ece_8_summed": 0.04109328570257781, + "nll_151": 4.830690229326394, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011612411252, + "ece_8_summed": 0.03788856275144198, + "nll_151": 4.750800036287824, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k1-seed44.json b/results/runs/majority-k1-seed44.json new file mode 100644 index 0000000..c3d5736 --- /dev/null +++ b/results/runs/majority-k1-seed44.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k1-seed44", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 1, + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "d69635e70893d70759a46fd0fa54561eae262653eca95898076348b9c660b96a" + }, + "logits": { + "file": "majority-k1-seed44.npz", + "sha256": "52a40757315fd9be80c2fd0cf5329f0b4c446ff25cf9a23ff714beab956155c9", + "bytes": 24098 + }, + "metrics": { + "temperature": 0.6826061908058604, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.012650220997595999, + "ece_8_summed": 0.003204722652724451, + "nll_151": 4.994998815055462, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 1.1779402661615812e-09, + "ece_8_summed": 2.9841140669617516e-10, + "nll_151": 4.991507764905244, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1622103382996488, + "ece_8_summed": 0.04109328570257781, + "nll_151": 4.830690229326394, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011612411252, + "ece_8_summed": 0.03788856275144198, + "nll_151": 4.750800036287824, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k10-seed42.json b/results/runs/majority-k10-seed42.json new file mode 100644 index 0000000..8891e61 --- /dev/null +++ b/results/runs/majority-k10-seed42.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k10-seed42", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 10, + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "2606cc878d85759a5885e1b9be35205023b156e6cf8a4eea85bd635c3b492e74" + }, + "logits": { + "file": "majority-k10-seed42.npz", + "sha256": "73c39d0e3e96319a1a3e339c4f7beeb57fa9c05ffbdda933ad50d8da0b47ce42", + "bytes": 24108 + }, + "metrics": { + "temperature": 0.5693234098482851, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.01586462143900791, + "ece_8_summed": 0.0040190374312154276, + "nll_151": 4.997606830056294, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 4.375323360561456e-09, + "ece_8_summed": 1.1084151652074326e-09, + "nll_151": 4.9915077649052435, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1654247387410607, + "ece_8_summed": 0.041907600481068785, + "nll_151": 4.860566276501883, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011292672944, + "ece_8_summed": 0.037888561941438276, + "nll_151": 4.750800020969453, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k10-seed43.json b/results/runs/majority-k10-seed43.json new file mode 100644 index 0000000..ba98682 --- /dev/null +++ b/results/runs/majority-k10-seed43.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k10-seed43", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 10, + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "072190bdfc028a2e0bc6ec85d39e26ae1b27854281c6226be57b8f8b9d20e165" + }, + "logits": { + "file": "majority-k10-seed43.npz", + "sha256": "10fd16db95cd9c12358fdb5a5d5d17c0ca0f6248c6f2c7506e03c8d30da5d650", + "bytes": 24108 + }, + "metrics": { + "temperature": 0.5693234098482851, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.01586462143900791, + "ece_8_summed": 0.0040190374312154276, + "nll_151": 4.997606830056294, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 4.375323360561456e-09, + "ece_8_summed": 1.1084151652074326e-09, + "nll_151": 4.9915077649052435, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1654247387410607, + "ece_8_summed": 0.041907600481068785, + "nll_151": 4.860566276501883, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011292672944, + "ece_8_summed": 0.037888561941438276, + "nll_151": 4.750800020969453, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k10-seed44.json b/results/runs/majority-k10-seed44.json new file mode 100644 index 0000000..099302c --- /dev/null +++ b/results/runs/majority-k10-seed44.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k10-seed44", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 10, + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "31a30630bb8f711a55173e8837eaf4500c2c5dcd5efe461b561a04e843cddd55" + }, + "logits": { + "file": "majority-k10-seed44.npz", + "sha256": "68305d812ce5fa3dc38d311014e4231cb19cb2a5fc25548011c3d448accfd491", + "bytes": 24107 + }, + "metrics": { + "temperature": 0.5693234098482851, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.01586462143900791, + "ece_8_summed": 0.0040190374312154276, + "nll_151": 4.997606830056294, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 4.375323360561456e-09, + "ece_8_summed": 1.1084151652074326e-09, + "nll_151": 4.9915077649052435, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1654247387410607, + "ece_8_summed": 0.041907600481068785, + "nll_151": 4.860566276501883, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011292672944, + "ece_8_summed": 0.037888561941438276, + "nll_151": 4.750800020969453, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k100-seed42.json b/results/runs/majority-k100-seed42.json new file mode 100644 index 0000000..2f769d0 --- /dev/null +++ b/results/runs/majority-k100-seed42.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k100-seed42", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 100, + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + "logits": { + "file": "majority-k100-seed42.npz", + "sha256": "45de4fe9a649263e6470be6fdbe46e8a4fceb5a97a0afe36c21062b7d6447fb4", + "bytes": 24111 + }, + "metrics": { + "temperature": 0.5693234098482851, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.01586462143900791, + "ece_8_summed": 0.0040190374312154276, + "nll_151": 4.997606830056294, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 4.375323360561456e-09, + "ece_8_summed": 1.1084151652074326e-09, + "nll_151": 4.9915077649052435, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1654247387410607, + "ece_8_summed": 0.041907600481068785, + "nll_151": 4.860566276501883, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011292672944, + "ece_8_summed": 0.037888561941438276, + "nll_151": 4.750800020969453, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k100-seed43.json b/results/runs/majority-k100-seed43.json new file mode 100644 index 0000000..147afe7 --- /dev/null +++ b/results/runs/majority-k100-seed43.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k100-seed43", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 100, + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + "logits": { + "file": "majority-k100-seed43.npz", + "sha256": "d57231c421661de1827d36c2b11d1a9cba0d5107f57447d639b25a70bcc46bb6", + "bytes": 24111 + }, + "metrics": { + "temperature": 0.5693234098482851, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.01586462143900791, + "ece_8_summed": 0.0040190374312154276, + "nll_151": 4.997606830056294, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 4.375323360561456e-09, + "ece_8_summed": 1.1084151652074326e-09, + "nll_151": 4.9915077649052435, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1654247387410607, + "ece_8_summed": 0.041907600481068785, + "nll_151": 4.860566276501883, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011292672944, + "ece_8_summed": 0.037888561941438276, + "nll_151": 4.750800020969453, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k100-seed44.json b/results/runs/majority-k100-seed44.json new file mode 100644 index 0000000..d215867 --- /dev/null +++ b/results/runs/majority-k100-seed44.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k100-seed44", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 100, + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + "logits": { + "file": "majority-k100-seed44.npz", + "sha256": "7422da060f87bd4325a17ef3f38948acf314349861f27c352038cbad2834c72b", + "bytes": 24111 + }, + "metrics": { + "temperature": 0.5693234098482851, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.01586462143900791, + "ece_8_summed": 0.0040190374312154276, + "nll_151": 4.997606830056294, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 4.375323360561456e-09, + "ece_8_summed": 1.1084151652074326e-09, + "nll_151": 4.9915077649052435, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1654247387410607, + "ece_8_summed": 0.041907600481068785, + "nll_151": 4.860566276501883, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011292672944, + "ece_8_summed": 0.037888561941438276, + "nll_151": 4.750800020969453, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k25-seed42.json b/results/runs/majority-k25-seed42.json new file mode 100644 index 0000000..d4bd999 --- /dev/null +++ b/results/runs/majority-k25-seed42.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k25-seed42", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 25, + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "51184b580bdbe58ab700d2545887a30c442950389b98497c3b7edaa7ca47174c" + }, + "logits": { + "file": "majority-k25-seed42.npz", + "sha256": "0847cf359ec51cb09bfec7ae8dfea24c09254963a5ea73a82875b4d1780c593e", + "bytes": 24108 + }, + "metrics": { + "temperature": 0.5742741151323764, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.015735644356524463, + "ece_8_summed": 0.003986363236986118, + "nll_151": 4.997480934744622, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 9.563552463842662e-09, + "ece_8_summed": 2.4227664596754295e-09, + "nll_151": 4.991507764905245, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.16529576165857726, + "ece_8_summed": 0.04187492628683945, + "nll_151": 4.859248693816038, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956010773850031, + "ece_8_summed": 0.03788856062708687, + "nll_151": 4.750799996113127, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k25-seed43.json b/results/runs/majority-k25-seed43.json new file mode 100644 index 0000000..8758ddf --- /dev/null +++ b/results/runs/majority-k25-seed43.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k25-seed43", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 25, + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "92f91be6e0948dbc01e82b81485c72829dd232443955fa0ae214b1451a0a704f" + }, + "logits": { + "file": "majority-k25-seed43.npz", + "sha256": "1d6145a48b53b954eba6a1d453b77f5eafc5f9cada582ff9012f7d39f1201ad5", + "bytes": 24108 + }, + "metrics": { + "temperature": 0.5742741151323764, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.015735644356524463, + "ece_8_summed": 0.003986363236986118, + "nll_151": 4.997480934744622, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 9.563552463842662e-09, + "ece_8_summed": 2.4227664596754295e-09, + "nll_151": 4.991507764905245, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.16529576165857726, + "ece_8_summed": 0.04187492628683945, + "nll_151": 4.859248693816038, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956010773850031, + "ece_8_summed": 0.03788856062708687, + "nll_151": 4.750799996113127, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k25-seed44.json b/results/runs/majority-k25-seed44.json new file mode 100644 index 0000000..380152f --- /dev/null +++ b/results/runs/majority-k25-seed44.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k25-seed44", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 25, + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "790cd7cf9eb24143276f6c59c7bac29c4a0b42f14bef38e9634e96253e6315fb" + }, + "logits": { + "file": "majority-k25-seed44.npz", + "sha256": "be5db7a3d7fda2f50097788b959e62a40acd27100a64ad4350f1527883183e1f", + "bytes": 24108 + }, + "metrics": { + "temperature": 0.5742741151323764, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.015735644356524463, + "ece_8_summed": 0.003986363236986118, + "nll_151": 4.997480934744622, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 9.563552463842662e-09, + "ece_8_summed": 2.4227664596754295e-09, + "nll_151": 4.991507764905245, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.16529576165857726, + "ece_8_summed": 0.04187492628683945, + "nll_151": 4.859248693816038, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956010773850031, + "ece_8_summed": 0.03788856062708687, + "nll_151": 4.750799996113127, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k5-seed42.json b/results/runs/majority-k5-seed42.json new file mode 100644 index 0000000..317d40d --- /dev/null +++ b/results/runs/majority-k5-seed42.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k5-seed42", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 5, + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "bfabf08971d3a16a502e25b2d5a7f1ce204504a526d5b3cb34fec9346dd8caa7" + }, + "logits": { + "file": "majority-k5-seed42.npz", + "sha256": "9e6bb6f163ed274f4c53aedd58604b2c1259d3c083384ea7fb2d49455f9db466", + "bytes": 24098 + }, + "metrics": { + "temperature": 0.5936924901783229, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.015220054559816641, + "ece_8_summed": 0.003855747155153494, + "nll_151": 4.996997167109025, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 1.934785408275186e-08, + "ece_8_summed": 4.901456324146736e-09, + "nll_151": 4.991507764905249, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.16478017186186944, + "ece_8_summed": 0.04174431020500685, + "nll_151": 4.854090744619826, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956009795419872, + "ece_8_summed": 0.037888558148397006, + "nll_151": 4.75079994923745, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k5-seed43.json b/results/runs/majority-k5-seed43.json new file mode 100644 index 0000000..ce99444 --- /dev/null +++ b/results/runs/majority-k5-seed43.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k5-seed43", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 5, + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "4e289193d9669c26c40d083246f6c34426bca5ff041ac526796a5f976ca9c638" + }, + "logits": { + "file": "majority-k5-seed43.npz", + "sha256": "3314ad9a70b585c6f4fccb9eb0bd56a5df209206e20b552c464727e10bbaba98", + "bytes": 24097 + }, + "metrics": { + "temperature": 0.5936924901783229, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.015220054559816641, + "ece_8_summed": 0.003855747155153494, + "nll_151": 4.996997167109025, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 1.934785408275186e-08, + "ece_8_summed": 4.901456324146736e-09, + "nll_151": 4.991507764905249, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.16478017186186944, + "ece_8_summed": 0.04174431020500685, + "nll_151": 4.854090744619826, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956009795419872, + "ece_8_summed": 0.037888558148397006, + "nll_151": 4.75079994923745, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k5-seed44.json b/results/runs/majority-k5-seed44.json new file mode 100644 index 0000000..e3c38fd --- /dev/null +++ b/results/runs/majority-k5-seed44.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k5-seed44", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 5, + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "ba57c52c7c416c611ba215bb10a393fba470ec511aaebad2b416914ca674988d" + }, + "logits": { + "file": "majority-k5-seed44.npz", + "sha256": "1da11415daed0f054fcb2477402f323bdcc4c0908abfebacfc116bc6f1ec5837", + "bytes": 24098 + }, + "metrics": { + "temperature": 0.5936924901783229, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.015220054559816641, + "ece_8_summed": 0.003855747155153494, + "nll_151": 4.996997167109025, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 1.934785408275186e-08, + "ece_8_summed": 4.901456324146736e-09, + "nll_151": 4.991507764905249, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.16478017186186944, + "ece_8_summed": 0.04174431020500685, + "nll_151": 4.854090744619826, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956009795419872, + "ece_8_summed": 0.037888558148397006, + "nll_151": 4.75079994923745, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k50-seed42.json b/results/runs/majority-k50-seed42.json new file mode 100644 index 0000000..d9a0f90 --- /dev/null +++ b/results/runs/majority-k50-seed42.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k50-seed42", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 50, + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "8a9a52f8ae41506d9847bd9ceba987edadc6bc3304487be22e664a95e1b9b18f" + }, + "logits": { + "file": "majority-k50-seed42.npz", + "sha256": "7c4872ed6ac1c993a1850633e26c44fac500db0efdc4923ea6a1ad1486a5ef97", + "bytes": 24110 + }, + "metrics": { + "temperature": 0.5693234098482851, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.01586462143900791, + "ece_8_summed": 0.0040190374312154276, + "nll_151": 4.997606830056294, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 4.375323360561456e-09, + "ece_8_summed": 1.1084151652074326e-09, + "nll_151": 4.9915077649052435, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1654247387410607, + "ece_8_summed": 0.041907600481068785, + "nll_151": 4.860566276501883, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011292672944, + "ece_8_summed": 0.037888561941438276, + "nll_151": 4.750800020969453, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k50-seed43.json b/results/runs/majority-k50-seed43.json new file mode 100644 index 0000000..6481a9f --- /dev/null +++ b/results/runs/majority-k50-seed43.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k50-seed43", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 50, + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "836ad6186d05a042ae00b3ab45efc560eb45f774644bf5e4f9b13a6e66d82982" + }, + "logits": { + "file": "majority-k50-seed43.npz", + "sha256": "ddd5754bd4acf617e268c6c68c49b3ab257bef78fb1e41e56cdb2d5508c3638e", + "bytes": 24110 + }, + "metrics": { + "temperature": 0.5693234098482851, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.01586462143900791, + "ece_8_summed": 0.0040190374312154276, + "nll_151": 4.997606830056294, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 4.375323360561456e-09, + "ece_8_summed": 1.1084151652074326e-09, + "nll_151": 4.9915077649052435, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1654247387410607, + "ece_8_summed": 0.041907600481068785, + "nll_151": 4.860566276501883, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011292672944, + "ece_8_summed": 0.037888561941438276, + "nll_151": 4.750800020969453, + "n": 5500 + } + } + } +} diff --git a/results/runs/majority-k50-seed44.json b/results/runs/majority-k50-seed44.json new file mode 100644 index 0000000..723b763 --- /dev/null +++ b/results/runs/majority-k50-seed44.json @@ -0,0 +1,82 @@ +{ + "run_name": "majority-k50-seed44", + "baseline": { + "name": "majority", + "version": "1", + "scores": "log share of each intent in the training sample, same for every query", + "params": {} + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 50, + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "5db5ce1ae1864ba63a4164a799b09a7c1d783a4acbd55efc587613d986752013" + }, + "logits": { + "file": "majority-k50-seed44.npz", + "sha256": "15276ebf3a368cb46ccb4beb03a529067369266cbfcfe1d5e6294e022d374e7c", + "bytes": 24110 + }, + "metrics": { + "temperature": 0.5693234098482851, + "validation": { + "raw": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.01586462143900791, + "ece_8_summed": 0.0040190374312154276, + "nll_151": 4.997606830056294, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.03225806451612903, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.24516129032258063, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 4.375323360561456e-09, + "ece_8_summed": 1.1084151652074326e-09, + "nll_151": 4.9915077649052435, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.1654247387410607, + "ece_8_summed": 0.041907600481068785, + "nll_151": 4.860566276501883, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.18181818181818182, + "oos_recall_8": 1.0, + "accuracy_8_summed": 0.20727272727272728, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.0, + "oos_recall_151": 1.0, + "ece_151": 0.14956011292672944, + "ece_8_summed": 0.037888561941438276, + "nll_151": 4.750800020969453, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k1-seed42.json b/results/runs/tfidf-centroid-k1-seed42.json new file mode 100644 index 0000000..28e2541 --- /dev/null +++ b/results/runs/tfidf-centroid-k1-seed42.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k1-seed42", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 1, + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "8f04aca1ffa32368a0e811795f70067f9d2b6d1551e10a3be5e08f5c954db166" + }, + "logits": { + "file": "tfidf-centroid-k1-seed42.npz", + "sha256": "94258de1803c08b0ba3fd2066c15b4814dfb701f499959b33319623e70d6089d", + "bytes": 2211462 + }, + "metrics": { + "temperature": 7.2126821471761895, + "validation": { + "raw": { + "accuracy_8": 0.5880645161290322, + "oos_recall_8": 0.01, + "accuracy_8_summed": 0.5935483870967742, + "oos_recall_8_summed": 0.01, + "in_scope_accuracy_150": 0.3506666666666667, + "oos_recall_151": 0.01, + "ece_151": 0.5730295681508771, + "ece_8_summed": 0.3467965742745178, + "nll_151": 12.2662424183968, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.5880645161290322, + "oos_recall_8": 0.01, + "accuracy_8_summed": 0.5958064516129032, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.3506666666666667, + "oos_recall_151": 0.01, + "ece_151": 0.07446184020822252, + "ece_8_summed": 0.10338653761208116, + "nll_151": 3.3042752347800066, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.4787272727272727, + "oos_recall_8": 0.009, + "accuracy_8_summed": 0.47963636363636364, + "oos_recall_8_summed": 0.009, + "in_scope_accuracy_150": 0.33666666666666667, + "oos_recall_151": 0.009, + "ece_151": 0.6359951430887093, + "ece_8_summed": 0.45749914430622796, + "nll_151": 14.190989190811969, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.4787272727272727, + "oos_recall_8": 0.009, + "accuracy_8_summed": 0.4852727272727273, + "oos_recall_8_summed": 0.002, + "in_scope_accuracy_150": 0.33666666666666667, + "oos_recall_151": 0.009, + "ece_151": 0.08054380075305323, + "ece_8_summed": 0.04435439687633225, + "nll_151": 3.6884129049619276, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k1-seed43.json b/results/runs/tfidf-centroid-k1-seed43.json new file mode 100644 index 0000000..da66586 --- /dev/null +++ b/results/runs/tfidf-centroid-k1-seed43.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k1-seed43", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 1, + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "4c659defaf9b9c6f28f4d5e67ef92b466627cdb200623552ba46ced1d2750c9c" + }, + "logits": { + "file": "tfidf-centroid-k1-seed43.npz", + "sha256": "d2d86e9f3d9cb80ca61f3323aa17dcb00ce80afbddf52beddf89da9ccb841380", + "bytes": 2387115 + }, + "metrics": { + "temperature": 7.088353756845146, + "validation": { + "raw": { + "accuracy_8": 0.5925806451612903, + "oos_recall_8": 0.04, + "accuracy_8_summed": 0.5958064516129032, + "oos_recall_8_summed": 0.04, + "in_scope_accuracy_150": 0.36133333333333334, + "oos_recall_151": 0.04, + "ece_151": 0.5722232399980438, + "ece_8_summed": 0.3498416966056108, + "nll_151": 11.855875032065574, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.5925806451612903, + "oos_recall_8": 0.04, + "accuracy_8_summed": 0.6180645161290322, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.36133333333333334, + "oos_recall_151": 0.04, + "ece_151": 0.0787436748655398, + "ece_8_summed": 0.12093237488009138, + "nll_151": 3.237398840702135, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.4812727272727273, + "oos_recall_8": 0.009, + "accuracy_8_summed": 0.4818181818181818, + "oos_recall_8_summed": 0.009, + "in_scope_accuracy_150": 0.33355555555555555, + "oos_recall_151": 0.009, + "ece_151": 0.6438042012196779, + "ece_8_summed": 0.45532873578711264, + "nll_151": 13.826544399332223, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.4812727272727273, + "oos_recall_8": 0.009, + "accuracy_8_summed": 0.5001818181818182, + "oos_recall_8_summed": 0.002, + "in_scope_accuracy_150": 0.33355555555555555, + "oos_recall_151": 0.009, + "ece_151": 0.08871463622656221, + "ece_8_summed": 0.03901105439262235, + "nll_151": 3.6498077993051714, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k1-seed44.json b/results/runs/tfidf-centroid-k1-seed44.json new file mode 100644 index 0000000..51e7ad2 --- /dev/null +++ b/results/runs/tfidf-centroid-k1-seed44.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k1-seed44", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 1, + "train_rows": 153, + "oos_train_rows": 3, + "train_sample_sha256": "d69635e70893d70759a46fd0fa54561eae262653eca95898076348b9c660b96a" + }, + "logits": { + "file": "tfidf-centroid-k1-seed44.npz", + "sha256": "fe1f9f4b8c84c652cec9717dd20fd94591d7134748764883360c9fe27ef5dbc5", + "bytes": 2290727 + }, + "metrics": { + "temperature": 7.30953027792008, + "validation": { + "raw": { + "accuracy_8": 0.5838709677419355, + "oos_recall_8": 0.02, + "accuracy_8_summed": 0.5858064516129032, + "oos_recall_8_summed": 0.02, + "in_scope_accuracy_150": 0.346, + "oos_recall_151": 0.02, + "ece_151": 0.5765391124418742, + "ece_8_summed": 0.35155902529227295, + "nll_151": 12.308933738901429, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.5838709677419355, + "oos_recall_8": 0.02, + "accuracy_8_summed": 0.5932258064516129, + "oos_recall_8_summed": 0.01, + "in_scope_accuracy_150": 0.346, + "oos_recall_151": 0.02, + "ece_151": 0.07471641381311003, + "ece_8_summed": 0.11099629441814356, + "nll_151": 3.3341003619588974, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.5036363636363637, + "oos_recall_8": 0.018, + "accuracy_8_summed": 0.5036363636363637, + "oos_recall_8_summed": 0.018, + "in_scope_accuracy_150": 0.36244444444444446, + "oos_recall_151": 0.018, + "ece_151": 0.6128819266621656, + "ece_8_summed": 0.4359871199965312, + "nll_151": 13.554576260934521, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.5036363636363637, + "oos_recall_8": 0.018, + "accuracy_8_summed": 0.5085454545454545, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.36244444444444446, + "oos_recall_151": 0.018, + "ece_151": 0.07493557962342517, + "ece_8_summed": 0.06730547394511692, + "nll_151": 3.550534520367745, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k10-seed42.json b/results/runs/tfidf-centroid-k10-seed42.json new file mode 100644 index 0000000..c9597a8 --- /dev/null +++ b/results/runs/tfidf-centroid-k10-seed42.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k10-seed42", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 10, + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "2606cc878d85759a5885e1b9be35205023b156e6cf8a4eea85bd635c3b492e74" + }, + "logits": { + "file": "tfidf-centroid-k10-seed42.npz", + "sha256": "efa61924b5913a37d54b9d4a3d13cd5be52fa35b56491dbfa9709fc8755b1aef", + "bytes": 4087246 + }, + "metrics": { + "temperature": 3.903436338051311, + "validation": { + "raw": { + "accuracy_8": 0.8493548387096774, + "oos_recall_8": 0.03, + "accuracy_8_summed": 0.8506451612903225, + "oos_recall_8_summed": 0.03, + "in_scope_accuracy_150": 0.724, + "oos_recall_151": 0.03, + "ece_151": 0.24053363341942896, + "ece_8_summed": 0.11743322783596202, + "nll_151": 3.4633775407324827, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8493548387096774, + "oos_recall_8": 0.03, + "accuracy_8_summed": 0.8516129032258064, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.724, + "oos_recall_151": 0.03, + "ece_151": 0.0832044613505549, + "ece_8_summed": 0.06343352448567725, + "nll_151": 1.433451826579588, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7272727272727273, + "oos_recall_8": 0.011, + "accuracy_8_summed": 0.7285454545454545, + "oos_recall_8_summed": 0.01, + "in_scope_accuracy_150": 0.7388888888888889, + "oos_recall_151": 0.011, + "ece_151": 0.32191034006113883, + "ece_8_summed": 0.2267505335006887, + "nll_151": 4.862965245431007, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7272727272727273, + "oos_recall_8": 0.011, + "accuracy_8_summed": 0.724909090909091, + "oos_recall_8_summed": 0.002, + "in_scope_accuracy_150": 0.7388888888888889, + "oos_recall_151": 0.011, + "ece_151": 0.07491550122330333, + "ece_8_summed": 0.041013603134895785, + "nll_151": 1.947161927616907, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k10-seed43.json b/results/runs/tfidf-centroid-k10-seed43.json new file mode 100644 index 0000000..7212cef --- /dev/null +++ b/results/runs/tfidf-centroid-k10-seed43.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k10-seed43", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 10, + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "072190bdfc028a2e0bc6ec85d39e26ae1b27854281c6226be57b8f8b9d20e165" + }, + "logits": { + "file": "tfidf-centroid-k10-seed43.npz", + "sha256": "aa4fc1ec5e6cdcac08cd404d778130f36725e6151a54f9ab2be795d38ed523e2", + "bytes": 4068984 + }, + "metrics": { + "temperature": 3.6381322836299153, + "validation": { + "raw": { + "accuracy_8": 0.8587096774193549, + "oos_recall_8": 0.05, + "accuracy_8_summed": 0.8609677419354839, + "oos_recall_8_summed": 0.04, + "in_scope_accuracy_150": 0.7283333333333334, + "oos_recall_151": 0.05, + "ece_151": 0.23295619911137727, + "ece_8_summed": 0.1047191134669071, + "nll_151": 2.9648993930330607, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8587096774193549, + "oos_recall_8": 0.05, + "accuracy_8_summed": 0.857741935483871, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.7283333333333334, + "oos_recall_151": 0.05, + "ece_151": 0.07101141086436225, + "ece_8_summed": 0.0462546328568032, + "nll_151": 1.301590790625417, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7361818181818182, + "oos_recall_8": 0.039, + "accuracy_8_summed": 0.7363636363636363, + "oos_recall_8_summed": 0.038, + "in_scope_accuracy_150": 0.7484444444444445, + "oos_recall_151": 0.039, + "ece_151": 0.3046706761824637, + "ece_8_summed": 0.2159555076558461, + "nll_151": 4.287756454896399, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7361818181818182, + "oos_recall_8": 0.039, + "accuracy_8_summed": 0.7316363636363636, + "oos_recall_8_summed": 0.019, + "in_scope_accuracy_150": 0.7484444444444445, + "oos_recall_151": 0.039, + "ece_151": 0.06709885185695615, + "ece_8_summed": 0.04057853290495013, + "nll_151": 1.8168140085760232, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k10-seed44.json b/results/runs/tfidf-centroid-k10-seed44.json new file mode 100644 index 0000000..03a22a9 --- /dev/null +++ b/results/runs/tfidf-centroid-k10-seed44.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k10-seed44", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 10, + "train_rows": 1525, + "oos_train_rows": 25, + "train_sample_sha256": "31a30630bb8f711a55173e8837eaf4500c2c5dcd5efe461b561a04e843cddd55" + }, + "logits": { + "file": "tfidf-centroid-k10-seed44.npz", + "sha256": "cfb9163965d84af33aef86d28145073b143ebb604bd9901d37b18d54d842db22", + "bytes": 4080654 + }, + "metrics": { + "temperature": 3.8125554938036714, + "validation": { + "raw": { + "accuracy_8": 0.853225806451613, + "oos_recall_8": 0.06, + "accuracy_8_summed": 0.8522580645161291, + "oos_recall_8_summed": 0.06, + "in_scope_accuracy_150": 0.7316666666666667, + "oos_recall_151": 0.06, + "ece_151": 0.23066597135332892, + "ece_8_summed": 0.11440650150609069, + "nll_151": 3.1561893874828937, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.853225806451613, + "oos_recall_8": 0.06, + "accuracy_8_summed": 0.8493548387096774, + "oos_recall_8_summed": 0.01, + "in_scope_accuracy_150": 0.7316666666666667, + "oos_recall_151": 0.06, + "ece_151": 0.07208216835903714, + "ece_8_summed": 0.04096393411807915, + "nll_151": 1.3180504507517907, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7301818181818182, + "oos_recall_8": 0.012, + "accuracy_8_summed": 0.730909090909091, + "oos_recall_8_summed": 0.01, + "in_scope_accuracy_150": 0.7515555555555555, + "oos_recall_151": 0.012, + "ece_151": 0.3127423318256793, + "ece_8_summed": 0.2281195568052044, + "nll_151": 4.825975491640957, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7301818181818182, + "oos_recall_8": 0.012, + "accuracy_8_summed": 0.7292727272727273, + "oos_recall_8_summed": 0.001, + "in_scope_accuracy_150": 0.7515555555555555, + "oos_recall_151": 0.012, + "ece_151": 0.06565298187104435, + "ece_8_summed": 0.050103204024827756, + "nll_151": 1.9165585081270513, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k100-seed42.json b/results/runs/tfidf-centroid-k100-seed42.json new file mode 100644 index 0000000..dcc6021 --- /dev/null +++ b/results/runs/tfidf-centroid-k100-seed42.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k100-seed42", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 100, + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + "logits": { + "file": "tfidf-centroid-k100-seed42.npz", + "sha256": "0ae4c761890317d76efc8d3b08a3920eab94633e32f710f0059cb7b96f82cb75", + "bytes": 4522480 + }, + "metrics": { + "temperature": 2.9156770942469437, + "validation": { + "raw": { + "accuracy_8": 0.8929032258064517, + "oos_recall_8": 0.2, + "accuracy_8_summed": 0.8925806451612903, + "oos_recall_8_summed": 0.19, + "in_scope_accuracy_150": 0.8216666666666667, + "oos_recall_151": 0.2, + "ece_151": 0.14692717655864765, + "ece_8_summed": 0.08046058992270064, + "nll_151": 1.6566815921732814, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8929032258064517, + "oos_recall_8": 0.2, + "accuracy_8_summed": 0.8909677419354839, + "oos_recall_8_summed": 0.11, + "in_scope_accuracy_150": 0.8216666666666667, + "oos_recall_151": 0.2, + "ece_151": 0.06557302511243435, + "ece_8_summed": 0.03263056436032928, + "nll_151": 0.8967562631534762, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7761818181818182, + "oos_recall_8": 0.096, + "accuracy_8_summed": 0.7741818181818182, + "oos_recall_8_summed": 0.086, + "in_scope_accuracy_150": 0.8362222222222222, + "oos_recall_151": 0.096, + "ece_151": 0.22427326296389014, + "ece_8_summed": 0.17927779561994664, + "nll_151": 2.6807427272026985, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7761818181818182, + "oos_recall_8": 0.096, + "accuracy_8_summed": 0.7627272727272727, + "oos_recall_8_summed": 0.025, + "in_scope_accuracy_150": 0.8362222222222222, + "oos_recall_151": 0.096, + "ece_151": 0.07369724591794735, + "ece_8_summed": 0.06770598629876846, + "nll_151": 1.3875105680093391, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k100-seed43.json b/results/runs/tfidf-centroid-k100-seed43.json new file mode 100644 index 0000000..6a29c56 --- /dev/null +++ b/results/runs/tfidf-centroid-k100-seed43.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k100-seed43", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 100, + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + "logits": { + "file": "tfidf-centroid-k100-seed43.npz", + "sha256": "01e56ce77413fd0e6dcb11398fdd0eb8024997983daf7bac5fea2a410d1bbe33", + "bytes": 4522481 + }, + "metrics": { + "temperature": 2.9156770942469437, + "validation": { + "raw": { + "accuracy_8": 0.8929032258064517, + "oos_recall_8": 0.2, + "accuracy_8_summed": 0.8925806451612903, + "oos_recall_8_summed": 0.19, + "in_scope_accuracy_150": 0.8216666666666667, + "oos_recall_151": 0.2, + "ece_151": 0.14692717655864765, + "ece_8_summed": 0.08046058992270064, + "nll_151": 1.6566815921732814, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8929032258064517, + "oos_recall_8": 0.2, + "accuracy_8_summed": 0.8909677419354839, + "oos_recall_8_summed": 0.11, + "in_scope_accuracy_150": 0.8216666666666667, + "oos_recall_151": 0.2, + "ece_151": 0.06557302511243435, + "ece_8_summed": 0.03263056436032928, + "nll_151": 0.8967562631534762, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7761818181818182, + "oos_recall_8": 0.096, + "accuracy_8_summed": 0.7741818181818182, + "oos_recall_8_summed": 0.086, + "in_scope_accuracy_150": 0.8362222222222222, + "oos_recall_151": 0.096, + "ece_151": 0.22427326296389014, + "ece_8_summed": 0.17927779561994664, + "nll_151": 2.6807427272026985, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7761818181818182, + "oos_recall_8": 0.096, + "accuracy_8_summed": 0.7627272727272727, + "oos_recall_8_summed": 0.025, + "in_scope_accuracy_150": 0.8362222222222222, + "oos_recall_151": 0.096, + "ece_151": 0.07369724591794735, + "ece_8_summed": 0.06770598629876846, + "nll_151": 1.3875105680093391, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k100-seed44.json b/results/runs/tfidf-centroid-k100-seed44.json new file mode 100644 index 0000000..655b7a3 --- /dev/null +++ b/results/runs/tfidf-centroid-k100-seed44.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k100-seed44", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 100, + "train_rows": 15250, + "oos_train_rows": 250, + "train_sample_sha256": "7a3ebd2cd2f985a26e815c229cc54add6763f783bee572c5e7948636c66c7553" + }, + "logits": { + "file": "tfidf-centroid-k100-seed44.npz", + "sha256": "ae69c656b6f00892ace610769c997b03dcfe588cb8568b4cd4e94bba9fee14bf", + "bytes": 4522482 + }, + "metrics": { + "temperature": 2.9156770942469437, + "validation": { + "raw": { + "accuracy_8": 0.8929032258064517, + "oos_recall_8": 0.2, + "accuracy_8_summed": 0.8925806451612903, + "oos_recall_8_summed": 0.19, + "in_scope_accuracy_150": 0.8216666666666667, + "oos_recall_151": 0.2, + "ece_151": 0.14692717655864765, + "ece_8_summed": 0.08046058992270064, + "nll_151": 1.6566815921732814, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8929032258064517, + "oos_recall_8": 0.2, + "accuracy_8_summed": 0.8909677419354839, + "oos_recall_8_summed": 0.11, + "in_scope_accuracy_150": 0.8216666666666667, + "oos_recall_151": 0.2, + "ece_151": 0.06557302511243435, + "ece_8_summed": 0.03263056436032928, + "nll_151": 0.8967562631534762, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7761818181818182, + "oos_recall_8": 0.096, + "accuracy_8_summed": 0.7741818181818182, + "oos_recall_8_summed": 0.086, + "in_scope_accuracy_150": 0.8362222222222222, + "oos_recall_151": 0.096, + "ece_151": 0.22427326296389014, + "ece_8_summed": 0.17927779561994664, + "nll_151": 2.6807427272026985, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7761818181818182, + "oos_recall_8": 0.096, + "accuracy_8_summed": 0.7627272727272727, + "oos_recall_8_summed": 0.025, + "in_scope_accuracy_150": 0.8362222222222222, + "oos_recall_151": 0.096, + "ece_151": 0.07369724591794735, + "ece_8_summed": 0.06770598629876846, + "nll_151": 1.3875105680093391, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k25-seed42.json b/results/runs/tfidf-centroid-k25-seed42.json new file mode 100644 index 0000000..c1f41f3 --- /dev/null +++ b/results/runs/tfidf-centroid-k25-seed42.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k25-seed42", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 25, + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "51184b580bdbe58ab700d2545887a30c442950389b98497c3b7edaa7ca47174c" + }, + "logits": { + "file": "tfidf-centroid-k25-seed42.npz", + "sha256": "be4fdf7bd9f789ab99de9042ffbc8b0d41d6cb152e08a7c079fd3bf623909a47", + "bytes": 4354875 + }, + "metrics": { + "temperature": 3.3845137053912495, + "validation": { + "raw": { + "accuracy_8": 0.8712903225806452, + "oos_recall_8": 0.1, + "accuracy_8_summed": 0.8729032258064516, + "oos_recall_8_summed": 0.09, + "in_scope_accuracy_150": 0.782, + "oos_recall_151": 0.1, + "ece_151": 0.18220922854063903, + "ece_8_summed": 0.09629164124995379, + "nll_151": 2.2812244444315355, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8712903225806452, + "oos_recall_8": 0.1, + "accuracy_8_summed": 0.8719354838709678, + "oos_recall_8_summed": 0.04, + "in_scope_accuracy_150": 0.782, + "oos_recall_151": 0.1, + "ece_151": 0.06571022143073199, + "ece_8_summed": 0.03518441737509228, + "nll_151": 1.0768981621627909, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7572727272727273, + "oos_recall_8": 0.047, + "accuracy_8_summed": 0.7578181818181818, + "oos_recall_8_summed": 0.045, + "in_scope_accuracy_150": 0.7986666666666666, + "oos_recall_151": 0.047, + "ece_151": 0.2645837618970529, + "ece_8_summed": 0.19770038871802117, + "nll_151": 3.627597907236036, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7572727272727273, + "oos_recall_8": 0.047, + "accuracy_8_summed": 0.7476363636363637, + "oos_recall_8_summed": 0.004, + "in_scope_accuracy_150": 0.7986666666666666, + "oos_recall_151": 0.047, + "ece_151": 0.07593663747224977, + "ece_8_summed": 0.05882721700695435, + "nll_151": 1.6275940948406245, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k25-seed43.json b/results/runs/tfidf-centroid-k25-seed43.json new file mode 100644 index 0000000..cbfa9ca --- /dev/null +++ b/results/runs/tfidf-centroid-k25-seed43.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k25-seed43", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 25, + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "92f91be6e0948dbc01e82b81485c72829dd232443955fa0ae214b1451a0a704f" + }, + "logits": { + "file": "tfidf-centroid-k25-seed43.npz", + "sha256": "c09e6879bf292df29c7fa24208687bdc43a2513f62033313cc100d5d6a1f0fa4", + "bytes": 4358905 + }, + "metrics": { + "temperature": 3.274604018333995, + "validation": { + "raw": { + "accuracy_8": 0.8858064516129033, + "oos_recall_8": 0.17, + "accuracy_8_summed": 0.8861290322580645, + "oos_recall_8_summed": 0.17, + "in_scope_accuracy_150": 0.7916666666666666, + "oos_recall_151": 0.17, + "ece_151": 0.17411198411151674, + "ece_8_summed": 0.0856487285211533, + "nll_151": 2.1487108014233818, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8858064516129033, + "oos_recall_8": 0.17, + "accuracy_8_summed": 0.8829032258064516, + "oos_recall_8_summed": 0.12, + "in_scope_accuracy_150": 0.7916666666666666, + "oos_recall_151": 0.17, + "ece_151": 0.05647034850617798, + "ece_8_summed": 0.03594428204392353, + "nll_151": 1.0380423982694558, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7634545454545455, + "oos_recall_8": 0.084, + "accuracy_8_summed": 0.7623636363636364, + "oos_recall_8_summed": 0.079, + "in_scope_accuracy_150": 0.8062222222222222, + "oos_recall_151": 0.084, + "ece_151": 0.2506006117805561, + "ece_8_summed": 0.1923517362008111, + "nll_151": 3.4375266065942176, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7634545454545455, + "oos_recall_8": 0.084, + "accuracy_8_summed": 0.7543636363636363, + "oos_recall_8_summed": 0.03, + "in_scope_accuracy_150": 0.8062222222222222, + "oos_recall_151": 0.084, + "ece_151": 0.06466559727937982, + "ece_8_summed": 0.05916400946762874, + "nll_151": 1.5706358315193498, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k25-seed44.json b/results/runs/tfidf-centroid-k25-seed44.json new file mode 100644 index 0000000..1281a67 --- /dev/null +++ b/results/runs/tfidf-centroid-k25-seed44.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k25-seed44", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 25, + "train_rows": 3813, + "oos_train_rows": 63, + "train_sample_sha256": "790cd7cf9eb24143276f6c59c7bac29c4a0b42f14bef38e9634e96253e6315fb" + }, + "logits": { + "file": "tfidf-centroid-k25-seed44.npz", + "sha256": "e7aa6e711a2a6b5322a78def95a33738c6bf0f80f6006710d203af2c9c3fdcd5", + "bytes": 4349783 + }, + "metrics": { + "temperature": 3.319341096373254, + "validation": { + "raw": { + "accuracy_8": 0.8819354838709678, + "oos_recall_8": 0.05, + "accuracy_8_summed": 0.882258064516129, + "oos_recall_8_summed": 0.05, + "in_scope_accuracy_150": 0.785, + "oos_recall_151": 0.05, + "ece_151": 0.18137780489921196, + "ece_8_summed": 0.08838374305507013, + "nll_151": 2.1839319783770734, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8819354838709678, + "oos_recall_8": 0.05, + "accuracy_8_summed": 0.8780645161290322, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.785, + "oos_recall_151": 0.05, + "ece_151": 0.053507371746975854, + "ece_8_summed": 0.03781975891510377, + "nll_151": 1.0511057708714628, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7467272727272727, + "oos_recall_8": 0.014, + "accuracy_8_summed": 0.7461818181818182, + "oos_recall_8_summed": 0.01, + "in_scope_accuracy_150": 0.8006666666666666, + "oos_recall_151": 0.014, + "ece_151": 0.27199132511947777, + "ece_8_summed": 0.21103582140244387, + "nll_151": 3.6248304193548506, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7467272727272727, + "oos_recall_8": 0.014, + "accuracy_8_summed": 0.7441818181818182, + "oos_recall_8_summed": 0.004, + "in_scope_accuracy_150": 0.8006666666666666, + "oos_recall_151": 0.014, + "ece_151": 0.07441269981393896, + "ece_8_summed": 0.06662534792302298, + "nll_151": 1.6295473997097685, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k5-seed42.json b/results/runs/tfidf-centroid-k5-seed42.json new file mode 100644 index 0000000..9d4512c --- /dev/null +++ b/results/runs/tfidf-centroid-k5-seed42.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k5-seed42", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 5, + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "bfabf08971d3a16a502e25b2d5a7f1ce204504a526d5b3cb34fec9346dd8caa7" + }, + "logits": { + "file": "tfidf-centroid-k5-seed42.npz", + "sha256": "cac96d18d9cad091f8dd3822554c9b94ae38e08d90688d161f52560791ccb63b", + "bytes": 3697662 + }, + "metrics": { + "temperature": 4.317316193644036, + "validation": { + "raw": { + "accuracy_8": 0.8064516129032258, + "oos_recall_8": 0.08, + "accuracy_8_summed": 0.8061290322580645, + "oos_recall_8_summed": 0.07, + "in_scope_accuracy_150": 0.6383333333333333, + "oos_recall_151": 0.08, + "ece_151": 0.3100585249946019, + "ece_8_summed": 0.15105667198914963, + "nll_151": 4.606297038006754, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8064516129032258, + "oos_recall_8": 0.08, + "accuracy_8_summed": 0.8029032258064516, + "oos_recall_8_summed": 0.01, + "in_scope_accuracy_150": 0.6383333333333333, + "oos_recall_151": 0.08, + "ece_151": 0.0949310428546961, + "ece_8_summed": 0.0662283720472021, + "nll_151": 1.7833977784645392, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.6785454545454546, + "oos_recall_8": 0.02, + "accuracy_8_summed": 0.6803636363636364, + "oos_recall_8_summed": 0.02, + "in_scope_accuracy_150": 0.6333333333333333, + "oos_recall_151": 0.02, + "ece_151": 0.40163652825850427, + "ece_8_summed": 0.27117833696294497, + "nll_151": 6.578538122770832, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.6785454545454546, + "oos_recall_8": 0.02, + "accuracy_8_summed": 0.6772727272727272, + "oos_recall_8_summed": 0.008, + "in_scope_accuracy_150": 0.6333333333333333, + "oos_recall_151": 0.02, + "ece_151": 0.08704747240311173, + "ece_8_summed": 0.038106876085140635, + "nll_151": 2.3455982864702203, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k5-seed43.json b/results/runs/tfidf-centroid-k5-seed43.json new file mode 100644 index 0000000..b79da71 --- /dev/null +++ b/results/runs/tfidf-centroid-k5-seed43.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k5-seed43", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 5, + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "4e289193d9669c26c40d083246f6c34426bca5ff041ac526796a5f976ca9c638" + }, + "logits": { + "file": "tfidf-centroid-k5-seed43.npz", + "sha256": "dbfdd8f7f5107d61cae3e48f2e7a99916ecdf6a094047b4053e65fe43bc3ec1d", + "bytes": 3708540 + }, + "metrics": { + "temperature": 4.174468001036531, + "validation": { + "raw": { + "accuracy_8": 0.8016129032258065, + "oos_recall_8": 0.03, + "accuracy_8_summed": 0.8012903225806451, + "oos_recall_8_summed": 0.03, + "in_scope_accuracy_150": 0.6406666666666667, + "oos_recall_151": 0.03, + "ece_151": 0.31088439058633943, + "ece_8_summed": 0.15930237251779414, + "nll_151": 4.410520986372844, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8016129032258065, + "oos_recall_8": 0.03, + "accuracy_8_summed": 0.8048387096774193, + "oos_recall_8_summed": 0.02, + "in_scope_accuracy_150": 0.6406666666666667, + "oos_recall_151": 0.03, + "ece_151": 0.06445230403523858, + "ece_8_summed": 0.060503888387318905, + "nll_151": 1.7333911590991615, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.6827272727272727, + "oos_recall_8": 0.004, + "accuracy_8_summed": 0.6836363636363636, + "oos_recall_8_summed": 0.004, + "in_scope_accuracy_150": 0.654, + "oos_recall_151": 0.004, + "ece_151": 0.3848934616426327, + "ece_8_summed": 0.264398760141408, + "nll_151": 6.300776042800418, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.6827272727272727, + "oos_recall_8": 0.004, + "accuracy_8_summed": 0.6810909090909091, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.654, + "oos_recall_151": 0.004, + "ece_151": 0.06915104839506833, + "ece_8_summed": 0.037475022218791516, + "nll_151": 2.313494233492547, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k5-seed44.json b/results/runs/tfidf-centroid-k5-seed44.json new file mode 100644 index 0000000..d240253 --- /dev/null +++ b/results/runs/tfidf-centroid-k5-seed44.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k5-seed44", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 5, + "train_rows": 763, + "oos_train_rows": 13, + "train_sample_sha256": "ba57c52c7c416c611ba215bb10a393fba470ec511aaebad2b416914ca674988d" + }, + "logits": { + "file": "tfidf-centroid-k5-seed44.npz", + "sha256": "e3e751ba605ba46a492f6f8ba0759c63b41239d46bf01f9513dc0d1d5db175ba", + "bytes": 3769932 + }, + "metrics": { + "temperature": 4.215820293968678, + "validation": { + "raw": { + "accuracy_8": 0.807741935483871, + "oos_recall_8": 0.0, + "accuracy_8_summed": 0.8083870967741935, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.6396666666666667, + "oos_recall_151": 0.0, + "ece_151": 0.3093881936657158, + "ece_8_summed": 0.15144637939277372, + "nll_151": 4.500056260462757, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.807741935483871, + "oos_recall_8": 0.0, + "accuracy_8_summed": 0.8106451612903226, + "oos_recall_8_summed": 0.0, + "in_scope_accuracy_150": 0.6396666666666667, + "oos_recall_151": 0.0, + "ece_151": 0.07490707867890814, + "ece_8_summed": 0.06367163664225134, + "nll_151": 1.7504197079172978, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.6907272727272727, + "oos_recall_8": 0.004, + "accuracy_8_summed": 0.6916363636363636, + "oos_recall_8_summed": 0.004, + "in_scope_accuracy_150": 0.6566666666666666, + "oos_recall_151": 0.004, + "ece_151": 0.38157102721104813, + "ece_8_summed": 0.2575576341726623, + "nll_151": 6.23489109584308, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.6907272727272727, + "oos_recall_8": 0.004, + "accuracy_8_summed": 0.688, + "oos_recall_8_summed": 0.001, + "in_scope_accuracy_150": 0.6566666666666666, + "oos_recall_151": 0.004, + "ece_151": 0.07162240016068266, + "ece_8_summed": 0.03600703807568193, + "nll_151": 2.2948940875785873, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k50-seed42.json b/results/runs/tfidf-centroid-k50-seed42.json new file mode 100644 index 0000000..3442ff6 --- /dev/null +++ b/results/runs/tfidf-centroid-k50-seed42.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k50-seed42", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 42, + "k_shot": 50, + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "8a9a52f8ae41506d9847bd9ceba987edadc6bc3304487be22e664a95e1b9b18f" + }, + "logits": { + "file": "tfidf-centroid-k50-seed42.npz", + "sha256": "27608c5f570bcb13a37d9cff167a0c2980f2915bc0fbce190c0027b0e52c3b93", + "bytes": 4457536 + }, + "metrics": { + "temperature": 3.071946446768046, + "validation": { + "raw": { + "accuracy_8": 0.8848387096774194, + "oos_recall_8": 0.15, + "accuracy_8_summed": 0.8858064516129033, + "oos_recall_8_summed": 0.15, + "in_scope_accuracy_150": 0.8053333333333333, + "oos_recall_151": 0.15, + "ece_151": 0.1674056799616045, + "ece_8_summed": 0.0874197556152422, + "nll_151": 1.8459158359832908, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8848387096774194, + "oos_recall_8": 0.15, + "accuracy_8_summed": 0.8829032258064516, + "oos_recall_8_summed": 0.08, + "in_scope_accuracy_150": 0.8053333333333333, + "oos_recall_151": 0.15, + "ece_151": 0.06526351955484859, + "ece_8_summed": 0.031214770075147735, + "nll_151": 0.9429634359166205, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7689090909090909, + "oos_recall_8": 0.083, + "accuracy_8_summed": 0.7685454545454545, + "oos_recall_8_summed": 0.081, + "in_scope_accuracy_150": 0.8224444444444444, + "oos_recall_151": 0.083, + "ece_151": 0.24073078929837638, + "ece_8_summed": 0.18850432624980928, + "nll_151": 3.0591103462357774, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7689090909090909, + "oos_recall_8": 0.083, + "accuracy_8_summed": 0.7607272727272727, + "oos_recall_8_summed": 0.031, + "in_scope_accuracy_150": 0.8224444444444444, + "oos_recall_151": 0.083, + "ece_151": 0.07553614874361701, + "ece_8_summed": 0.06674804479836834, + "nll_151": 1.471361791538036, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k50-seed43.json b/results/runs/tfidf-centroid-k50-seed43.json new file mode 100644 index 0000000..ad0ce41 --- /dev/null +++ b/results/runs/tfidf-centroid-k50-seed43.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k50-seed43", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 43, + "k_shot": 50, + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "836ad6186d05a042ae00b3ab45efc560eb45f774644bf5e4f9b13a6e66d82982" + }, + "logits": { + "file": "tfidf-centroid-k50-seed43.npz", + "sha256": "9f8eed2728260b4b58e05b7e7d70b79b4b5f74d6b9f7a01decc8cff13e42aa2f", + "bytes": 4461339 + }, + "metrics": { + "temperature": 3.107605063695964, + "validation": { + "raw": { + "accuracy_8": 0.8932258064516129, + "oos_recall_8": 0.16, + "accuracy_8_summed": 0.892258064516129, + "oos_recall_8_summed": 0.14, + "in_scope_accuracy_150": 0.8093333333333333, + "oos_recall_151": 0.16, + "ece_151": 0.16309442204694385, + "ece_8_summed": 0.08077321852975211, + "nll_151": 1.877481684141421, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8932258064516129, + "oos_recall_8": 0.16, + "accuracy_8_summed": 0.8903225806451613, + "oos_recall_8_summed": 0.08, + "in_scope_accuracy_150": 0.8093333333333333, + "oos_recall_151": 0.16, + "ece_151": 0.07230114350278577, + "ece_8_summed": 0.04327708623441586, + "nll_151": 0.9602776625207111, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7612727272727273, + "oos_recall_8": 0.049, + "accuracy_8_summed": 0.761090909090909, + "oos_recall_8_summed": 0.046, + "in_scope_accuracy_150": 0.8135555555555556, + "oos_recall_151": 0.049, + "ece_151": 0.25460152901183164, + "ece_8_summed": 0.19570441424496632, + "nll_151": 3.1277935942428523, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7612727272727273, + "oos_recall_8": 0.049, + "accuracy_8_summed": 0.7532727272727273, + "oos_recall_8_summed": 0.009, + "in_scope_accuracy_150": 0.8135555555555556, + "oos_recall_151": 0.049, + "ece_151": 0.07836733106525698, + "ece_8_summed": 0.06914498014263623, + "nll_151": 1.503566647047933, + "n": 5500 + } + } + } +} diff --git a/results/runs/tfidf-centroid-k50-seed44.json b/results/runs/tfidf-centroid-k50-seed44.json new file mode 100644 index 0000000..16566fa --- /dev/null +++ b/results/runs/tfidf-centroid-k50-seed44.json @@ -0,0 +1,88 @@ +{ + "run_name": "tfidf-centroid-k50-seed44", + "baseline": { + "name": "tfidf-centroid", + "version": "1", + "scores": "100 * cosine(query tf-idf, L2-normalised intent centroid); vectorizer fit on the training sample, word 1-2 grams, sublinear tf", + "params": { + "ngram_range": [ + 1, + 2 + ], + "sublinear_tf": true + } + }, + "environment": { + "python": "3.12.12", + "numpy": "2.5.3", + "scikit-learn": "1.9.1", + "dataset_revision": "155b9c710419136e17307b80d0a13e68cd46b4ec" + }, + "training": { + "seed": 44, + "k_shot": 50, + "train_rows": 7625, + "oos_train_rows": 125, + "train_sample_sha256": "5db5ce1ae1864ba63a4164a799b09a7c1d783a4acbd55efc587613d986752013" + }, + "logits": { + "file": "tfidf-centroid-k50-seed44.npz", + "sha256": "3fa2c12a4a881fe0c9148c86acc9be6edd315d968621aecf6f7936c0927d79ab", + "bytes": 4455620 + }, + "metrics": { + "temperature": 3.153115924527087, + "validation": { + "raw": { + "accuracy_8": 0.8832258064516129, + "oos_recall_8": 0.13, + "accuracy_8_summed": 0.8825806451612903, + "oos_recall_8_summed": 0.11, + "in_scope_accuracy_150": 0.8096666666666666, + "oos_recall_151": 0.13, + "ece_151": 0.1584927923123098, + "ece_8_summed": 0.08663902496089831, + "nll_151": 1.9066840785385568, + "n": 3100 + }, + "calibrated": { + "accuracy_8": 0.8832258064516129, + "oos_recall_8": 0.13, + "accuracy_8_summed": 0.8793548387096775, + "oos_recall_8_summed": 0.03, + "in_scope_accuracy_150": 0.8096666666666666, + "oos_recall_151": 0.13, + "ece_151": 0.06347489274195099, + "ece_8_summed": 0.03555324220861267, + "nll_151": 0.9728950291884839, + "n": 3100 + } + }, + "test": { + "raw": { + "accuracy_8": 0.7634545454545455, + "oos_recall_8": 0.053, + "accuracy_8_summed": 0.763090909090909, + "oos_recall_8_summed": 0.047, + "in_scope_accuracy_150": 0.8204444444444444, + "oos_recall_151": 0.053, + "ece_151": 0.24567130137449797, + "ece_8_summed": 0.1922184749760928, + "nll_151": 3.1019700014162344, + "n": 5500 + }, + "calibrated": { + "accuracy_8": 0.7634545454545455, + "oos_recall_8": 0.053, + "accuracy_8_summed": 0.756, + "oos_recall_8_summed": 0.015, + "in_scope_accuracy_150": 0.8204444444444444, + "oos_recall_151": 0.053, + "ece_151": 0.0676270745964987, + "ece_8_summed": 0.0650824905950811, + "nll_151": 1.4896602728258153, + "n": 5500 + } + } + } +}