Fix MMMU-Pro evaluation (EvolvingLMMs-Lab#296)

zhijian-liu · web-flow · commit 09d15c35687b · 2024-10-05T23:46:44.000+08:00
diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro.yaml
@@ -2,4 +2,4 @@ group: mmmu_pro
 task:
 - mmmu_pro_vision
 # - mmmu_pro_composite # removing composite task in formal MMMU-Pro evaluation
-- mmmu_pro_original
+- mmmu_pro_standard
diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard.yaml
@@ -1,6 +1,6 @@
-task: "mmmu_pro_original"
+task: "mmmu_pro_standard"
 dataset_path: MMMU/MMMU_Pro
-dataset_name: original
+dataset_name: standard
 test_split: test
 output_type: generate_until
 doc_to_visual: !function utils.mmmu_pro_doc_to_visual
diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_standard_cot.yaml
@@ -1,6 +1,6 @@
-task: "mmmu_pro_original_cot"
+task: "mmmu_pro_standard_cot"
 dataset_path: MMMU/MMMU_Pro
-dataset_name: original
+dataset_name: standard
 test_split: test
 output_type: generate_until
 doc_to_visual: !function utils.mmmu_pro_doc_to_visual
diff --git a/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml b/lmms_eval/tasks/mmmu_pro/mmmu_pro_vision.yaml
@@ -4,7 +4,7 @@ dataset_name: vision
 test_split: test
 output_type: generate_until
 doc_to_visual: !function utils.mmmu_pro_doc_to_visual
-doc_to_text: "Answer with the option letter from the given choices directly. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options."
+doc_to_text: "Answer with the option letter from the given choices directly."
 doc_to_target: "{{answer}}"
 # The return value of process_results will be used by metrics
 process_results: !function utils.mmmu_pro_process_results
