[WIP] Add Gemini and GPT-Audio

README.md

@@ -57,6 +57,10 @@ and set `${H2T_DATADIR}` to the directory containing the corresponding audio fil
 - **Emotion**: [`emotiontalk`](manifests/emotiontalk/README.md), [`mexpresso`](manifests/mexpresso/README.md)
 - **Long-Form**: [`acl6060-long`](manifests/acl6060-long/README.md), [`acl6060-short`](manifests/acl6060-short/README.md), [`mcif-long`](manifests/mcif-long/README.md), [`mcif-short`](manifests/mcif-short/README.md)
 
+Optionally: 
+- Test if all audio files in dataset exist in the right location: `tests/test_dataset.py` 
+- Count testset statistics: `tests/stat_dataset.py`
+
 ### 2. Run inference
 
 Run inference with the following command:
@@ -70,6 +74,11 @@ Run inference with the following command:
 The full list of supported models can be obtained with `python infer.py -h`.
 Supported benchmarks are listed above, while benchmark-specific language coverage is documented in the corresponding READMEs.
 
+To use the Gemini API, you must set the environment variable with your API key:
+```
+export GEMINI_API_KEY=<your-api-key>
+```
+
 ### 3. Run evaluation
 
 After generating model outputs, run the evaluation suite using the scripts in the `evaluation/` directory.

evaluation/output_evals/covost2/gemini-2.5-flash/de_en/results_summary.jsonl

@@ -0,0 +1 @@
+{"SacreBLEU": 40.031, "chrF": 64.3759, "XCOMET": 0.9059, "XCOMET-QE": 0.8795, "RefMetricX_24": 3.635, "QEMetricX_24": 3.6569, "LinguaPy": 4.4482, "RefMetricX_24-Strict-linguapy": 4.5578, "QEMetricX_24-Strict-linguapy": 4.5728, "XCOMET-Strict-linguapy": 0.8677, "XCOMET-QE-Strict-linguapy": 0.8424}

evaluation/output_evals/covost2/gemini-2.5-flash/en_de/results_summary.jsonl

@@ -0,0 +1 @@
+{"SacreBLEU": 34.9453, "chrF": 60.7662, "XCOMET": 0.9147, "XCOMET-QE": 0.9177, "RefMetricX_24": 2.1869, "QEMetricX_24": 2.1879, "LinguaPy": 1.526, "RefMetricX_24-Strict-linguapy": 2.4985, "QEMetricX_24-Strict-linguapy": 2.4942, "XCOMET-Strict-linguapy": 0.9031, "XCOMET-QE-Strict-linguapy": 0.9061}

evaluation/output_evals/covost2/gemini-2.5-flash/en_es/results_summary.jsonl

@@ -0,0 +1 @@
+{"XCOMET-QE": 0.8569, "QEMetricX_24": 3.566, "LinguaPy": 4.0757, "QEMetricX_24-Strict-linguapy": 4.3974, "XCOMET-QE-Strict-linguapy": 0.8284}

evaluation/output_evals/covost2/gemini-2.5-flash/en_fr/results_summary.jsonl

@@ -0,0 +1 @@
+{"XCOMET-QE": 0.8171, "QEMetricX_24": 3.6918, "LinguaPy": 1.9767, "QEMetricX_24-Strict-linguapy": 4.0718, "XCOMET-QE-Strict-linguapy": 0.8043}

evaluation/output_evals/covost2/gemini-2.5-flash/en_it/results_summary.jsonl

@@ -0,0 +1 @@
+{"XCOMET-QE": 0.8376, "QEMetricX_24": 3.5933, "LinguaPy": 3.8246, "QEMetricX_24-Strict-linguapy": 4.3602, "XCOMET-QE-Strict-linguapy": 0.8124}

evaluation/output_evals/covost2/gemini-2.5-flash/en_nl/results_summary.jsonl

@@ -0,0 +1 @@
+{"XCOMET-QE": 0.8926, "QEMetricX_24": 3.3753, "LinguaPy": 3.8954, "QEMetricX_24-Strict-linguapy": 4.1708, "XCOMET-QE-Strict-linguapy": 0.8612}

evaluation/output_evals/covost2/gemini-2.5-flash/en_pt/results_summary.jsonl

@@ -0,0 +1 @@
+{"XCOMET-QE": 0.8656, "QEMetricX_24": 4.1375, "LinguaPy": 3.9019, "QEMetricX_24-Strict-linguapy": 4.8819, "XCOMET-QE-Strict-linguapy": 0.8372}

evaluation/output_evals/covost2/gemini-2.5-flash/en_zh/results_summary.jsonl

@@ -0,0 +1 @@
+{"SacreBLEU": 40.759, "chrF": 33.5414, "XCOMET": 0.8681, "XCOMET-QE": 0.8493, "RefMetricX_24": 2.0848, "QEMetricX_24": 2.2457, "LinguaPy": 0.1095, "RefMetricX_24-Strict-linguapy": 2.1023, "QEMetricX_24-Strict-linguapy": 2.2589, "XCOMET-Strict-linguapy": 0.8675, "XCOMET-QE-Strict-linguapy": 0.8489}

evaluation/output_evals/covost2/gemini-2.5-flash/es_en/results_summary.jsonl

@@ -0,0 +1 @@
+{"SacreBLEU": 42.4109, "chrF": 66.8989, "XCOMET": 0.9022, "XCOMET-QE": 0.8847, "RefMetricX_24": 3.639, "QEMetricX_24": 3.4421, "LinguaPy": 4.4777, "RefMetricX_24-Strict-linguapy": 4.5582, "QEMetricX_24-Strict-linguapy": 4.3559, "XCOMET-Strict-linguapy": 0.8641, "XCOMET-QE-Strict-linguapy": 0.8476}

evaluation/output_evals/covost2/gemini-2.5-flash/it_en/results_summary.jsonl

@@ -0,0 +1 @@
+{"SacreBLEU": 39.2412, "chrF": 65.1239, "XCOMET": 0.8851, "XCOMET-QE": 0.861, "RefMetricX_24": 3.6573, "QEMetricX_24": 3.4602, "LinguaPy": 5.2955, "RefMetricX_24-Strict-linguapy": 4.7895, "QEMetricX_24-Strict-linguapy": 4.5939, "XCOMET-Strict-linguapy": 0.8415, "XCOMET-QE-Strict-linguapy": 0.8169}

evaluation/output_evals/covost2/gemini-2.5-flash/pt_en/results_summary.jsonl

@@ -0,0 +1 @@
+{"SacreBLEU": 54.4028, "chrF": 73.1512, "XCOMET": 0.9101, "XCOMET-QE": 0.871, "RefMetricX_24": 2.7843, "QEMetricX_24": 2.9325, "LinguaPy": 2.262, "RefMetricX_24-Strict-linguapy": 3.2472, "QEMetricX_24-Strict-linguapy": 3.3935, "XCOMET-Strict-linguapy": 0.8919, "XCOMET-QE-Strict-linguapy": 0.8542}

evaluation/output_evals/covost2/gemini-2.5-flash/zh_en/results_summary.jsonl

@@ -0,0 +1 @@
+{"SacreBLEU": 23.4225, "chrF": 52.5903, "XCOMET": 0.7809, "XCOMET-QE": 0.8179, "RefMetricX_24": 4.683, "QEMetricX_24": 3.6345, "LinguaPy": 11.7191, "RefMetricX_24-Strict-linguapy": 6.9063, "QEMetricX_24-Strict-linguapy": 6.0799, "XCOMET-Strict-linguapy": 0.7026, "XCOMET-QE-Strict-linguapy": 0.7296}

evaluation/output_evals/europarl_st/gemini-2.5-flash/de_en/results_summary.jsonl

@@ -0,0 +1 @@
+{"SacreBLEU": 29.5399, "chrF": 57.5177, "XCOMET": 0.8715, "XCOMET-QE": 0.8654, "RefMetricX_24": 4.5573, "QEMetricX_24": 4.0121, "LinguaPy": 0.3801, "RefMetricX_24-Strict-linguapy": 4.6187, "QEMetricX_24-Strict-linguapy": 4.0835, "XCOMET-Strict-linguapy": 0.8686, "XCOMET-QE-Strict-linguapy": 0.8625}

evaluation/output_evals/europarl_st/gemini-2.5-flash/en_de/results_summary.jsonl

@@ -0,0 +1 @@
+{"SacreBLEU": 32.1047, "chrF": 62.0079, "XCOMET": 0.9429, "XCOMET-QE": 0.95, "RefMetricX_24": 1.8137, "QEMetricX_24": 1.6219, "LinguaPy": 0.0798, "RefMetricX_24-Strict-linguapy": 1.8336, "QEMetricX_24-Strict-linguapy": 1.6416, "XCOMET-Strict-linguapy": 0.9421, "XCOMET-QE-Strict-linguapy": 0.9492}